# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

Class(RollingPointers, rec(
    __call__ := (self, code, opts) >> CopyFields(self, rec(opts := opts)).apply(code),

    _linearPatterns := Set([[], [mul]]),

    _contains    := (v, U) -> CollectNR(U, @.cond(u->u=v))<>[],
    _linear_exp  := (self, e, v) >> Length(Collect(e, @@.cond((x, cx)-> x=v and (List(cx.parents, ObjId) in self._linearPatterns))))=1,

    _basePtr  := (summands) -> let( p := Filtered(summands, e->IsPtrT(e.t) or IsArrayT(e.t)), Checked(Length(p)=1, p[1]) ),

    # _collectPtrs(<l>) -- <l> loop
    _collectPtrs := meth(self, l)
        local free, vptr, pattern, ptradds, ptrs, e, p, d, id, clusters, loopinv, i;

        # Collecting innermost ponter arithmetic additions.
        # Index computation expression expected to be in normalized form.
        pattern := @@(1, [add], (x, cx) -> IsPtrT(x.t) and (Filtered(cx.parents, IsLoop)=[] or Last(Filtered(cx.parents, IsLoop))=l));

        ptradds := Collect(l, pattern);
        ptradds := Filtered(ptradds, x -> Collect(x.args, pattern)=[]);

        ptrs := tab();

        free := Set(l.free() :: [l.var]);
        # extracting base pointer and common subexpression with this pointer.
        # relying on expression print method to get string id.
        for e in ptradds do
            p  := self._basePtr(e.args);
            # we may have buffer defined in the loop referenced by loop variable
            vptr := Collect(p, var);
            if vptr=[] or Intersection(vptr, free)<>[] then
                id := StringPrint(p);
                if IsBound(ptrs.(id)) then
                    ptrs.(id).cmn := Checked(ptrs.(id).ptr = p, Intersection(ptrs.(id).cmn, Set(e.args)));
                else
                    ptrs.(id) := rec( ptr := p, cmn := Set(e.args), offs := [] );
                fi;
            fi;
        od;
        # make sure common expression doesn't have local loop variables
        for id in UserNSFields(ptrs) do
            ptrs.(id).cmn := Filtered(ptrs.(id).cmn, a -> IsSubset(free, Collect(a, var)));
        od;
        # preparing offsets list for each base expression.
        #
        # offset record: rec(
        #    ref  := <full original expression>,
        #    tail := <offset from common base expression, as summands list>
        # )
        for e in ptradds do
            id := StringPrint(self._basePtr(e.args));
            if IsBound(ptrs.(id)) then
                d := Difference(e.args, ptrs.(id).cmn);
                Add(ptrs.(id).offs, rec( ref := e, tail := d ));
            fi;
        od;

        # when loop variable is not in common subexpression 'cmn' but still in 'tail'
        # try to split pointer so that variable goes to 'cmn' in each resulting pointer
        for id in UserNSFields(ptrs) do
            p := ptrs.(id);
            if not self._contains(l.var, p.cmn) then
                [ clusters, loopinv ] := SplitBy(ConcatList(p.offs, o -> o.tail), s -> self._linear_exp(s, l.var));
                if ForAll(loopinv, s -> not self._contains(l.var, s)) then
                    clusters := Set(clusters);
                    # NOTE: hardcoded max number of resulting rolling pointers
                    if Length(clusters)<=4 then
                        for i in [1..Length(clusters)] do
                            ptrs.(id :: "_" :: StringInt(i)) := rec(
                                ptr  := p.ptr,
                                cmn  := Set( p.cmn :: [clusters[i]] ),
                                # each original <offs> can contain only one expression from <clusters>
                                # because previousely summands were grouped by loop variables
                                offs := List(Filtered(p.offs, o -> clusters[i] in o.tail),
                                          o -> rec( ref := o.ref, tail := RemoveList(o.tail, clusters[i]) )),
                            );
                        od;
                        Unbind(ptrs.(id));
                    fi;
                fi;
            fi;
        od;

        return ptrs;
    end,

    _offsetsInit := function(vars, init, coeff)
        vars.linv := vars.linv :: vars.offs;
        init.linv := init.linv :: List( [1..Length(coeff)], i -> assign(vars.offs[i], vars.strides*coeff[i]));
    end,

    # cost is number of rolling pointers + their stride + number of offsets from rolling pointers
    _offsetsCost := (self, k) >> Cond(Length(k.rp_coeff)>4, 100000, self.rpCost*Length(k.rp_coeff) + 1 + self.coeffCost*Length(k.coeff)),

    # default cost of rolling pointer and coefficient
    rpCost    := 3,
    coeffCost := 1,

    # includeValues
    includeValues := false,

    _strideGrp := s -> CondPat( s, @(1, Value),       V(1),
                                   [mul, Value, @],   s.args[2],
                                # else
                                   s ),
    _strideVal := s -> CondPat( s, @(1, Value),       s.v,
                                   [mul, Value, @],   s.args[1].v,
                                # else
                                   1 ),

    _getStrides := meth(self, l, offs)
        local s, strides, offsets, free;

        if self.includeValues then
            s := ConcatList(offs, e -> e.tail);
        else
            s := ConcatList(offs, e -> Filtered(e.tail, x -> not (x _is Value)));
        fi;
        # ignore expressions with loop-local variables
        free := Set(l.free() :: [l.var]);
        s := Filtered(s, a -> IsSubset(free, Collect(a, var)));

        # get strides vector
        strides := Filtered(GroupList(s, self._strideGrp), e -> Length(e[2])>1);
        if strides<>[] then
            strides := TransposedMat(strides)[1];
        fi;

        # get stride coefficients for each offset + leftover expression
        offsets := TransposedMat(List(offs, function(e)
            local c, leftovers, x, p;
            leftovers := [];
            c := Replicate(Length(strides), 0);
            for x in e.tail do
                p := Position(strides, self._strideGrp(x));
                if p<>false then
                    c[p] := self._strideVal(x);
                else
                    Add(leftovers, x);
                fi;
            od;
            return [c, rec(ref := e.ref, leftovers := leftovers)];
        end));

        return [strides, offsets[1], offsets[2]];
    end,

    # reduce <coeff> offsets list to coefficients relative to <k> "evenly" spaced rolling pointers
    _k_func := function(k, coeff)
        local n, rp, offs, i;
        n     := Length(coeff);
        rp    := List([0..k-1], i -> QuoInt(n*i, k)+1) :: [n+1];
        offs := [];
        for i in [1..k] do
            offs := UnionSet(offs, List(coeff{[rp[i]+1..rp[i+1]-1]}, c -> c - coeff[rp[i]]));
        od;
        return rec(
            rp_coeff := List(DropLast(rp, 1), i -> coeff[i]),
            coeff    := offs,
        );
    end,

    # finds best split [<rolling ptrs>, <offsets>] for <coeff> offsets list.
    # returns record:
    #    .rp_coeff - list of coefficients for rolling pointers
    #    .coeff    - reduced <coeff> list so that every original <coeff>
    #                can be expressed as some <rp_coeff> plus some <coeff>
    #                from this list.
    #
    # Brute force, _offsetsCost can be redefined

    _bestSplit := meth(self, coeff)
        local scoeff, min_k, min_cost, i, j, k, o, cost;
        scoeff := Set(coeff);
        min_k := self._k_func(1, scoeff);
        min_cost := self._offsetsCost(min_k);
        for i in [1..Length(scoeff[1])] do
            o := Sort(ShallowCopy(scoeff), (a,b) -> Cond(a[i]=b[i], a<b, a[i]<b[i]));
            for j in [1..Length(Set(List(o, e -> e[i])))] do
                k := self._k_func(j, o);
                cost := self._offsetsCost(k);
                if cost < min_cost then
                    min_k := k;
                    min_cost := cost;
                fi;
            od;
        od;
        return min_k;
    end,

    # find indices in <offsets> of a given <offs> (not including crossed-out in <refs>)
    _offsetsIndices := (offsets, offs, refs) -> Filtered([1..Length(offsets)], i -> offsets[i]=offs and refs[i].ref<>false),

    # add mapping pair to <map> and cross-out index.
    _mapRecord := function(map, rem, summands)
        Add(map, rec(
            ref := rem.ref,
            subst := ApplyFunc(add, summands :: rem.leftovers)
        ));
        rem.ref := false;
    end,

    # split pointer expressions into rolling pointers + offsets
    _splitPtr := meth(self, ptr, loop, vars, init, incr, map)
        local strides, offsets_coeff, offsets_rem, cfg, i, j, k, ld, li, p;

        [strides, offsets_coeff, offsets_rem] := self._getStrides(loop, ptr.offs);

        cfg := self._bestSplit(offsets_coeff);

        # temporary variables lists.
        # Doing CSE is smarter as we may have same stride and offset computation expressions
        # for other pointers. I don't have this situation in DFTs though.
        vars.rp      := List(cfg.rp_coeff, e -> var.fresh_t("rp",   Cond(IsArrayT(ptr.ptr.t), ptr.ptr.t.toPtrType(), ptr.ptr.t)));
        vars.offs    := List(cfg.coeff,    e -> var.fresh_t("offs", TInt));
        vars.strides := List(strides,      e -> var.fresh_t("s",    TInt));
        vars.inc     := var.fresh_t("inc", TInt);

        # vars.offs is not added here as they might be declared in or outside of the loop
        # this is done by initOffsets method later
        vars.linv    := vars.linv :: vars.rp :: vars.strides :: [vars.inc];

        # preparing mapping from original pointer expressions to rolling pointers expressions
        for i in [1..Length(cfg.rp_coeff)] do
            # for each expression which correspond to this rolling pointer
            for k in self._offsetsIndices(offsets_coeff, cfg.rp_coeff[i], offsets_rem) do
                self._mapRecord(map, offsets_rem[k], [vars.rp[i]]);
            od;

            for j in [1..Length(cfg.coeff)] do
                # for each expression which correspond to this offset from current rolling pointer
                for k in self._offsetsIndices(offsets_coeff, cfg.rp_coeff[i]+cfg.coeff[j], offsets_rem) do
                    self._mapRecord(map, offsets_rem[k], [vars.rp[i], vars.offs[j]]);
                od;
            od;
        od;

        Checked(ForAll(offsets_rem, e -> e.ref=false), "paranoid");

        # <ld> loop var dependent summands, <li> - loop invariant summands
        [ld, li] := SplitBy(ptr.cmn, e -> self._contains(loop.var, e));

        # initialization code
        init.strides := List([1..Length(strides)], i -> assign(vars.strides[i], strides[i]));
        init.rp      := List([1..Length(cfg.rp_coeff)], i -> assign(vars.rp[i], SReduce(ApplyFunc(add, li :: SubstVars( Copy(ld), rec( (loop.var.id) := V(0))) :: Cond(vars.strides<>[], [vars.strides*cfg.rp_coeff[i]], [])), self.opts)));
        init.inc     := assign(vars.inc, SReduce(ApplyFunc(add, SubstVars( Copy(ld), rec( (loop.var.id) := V(1)))), self.opts));

        init.linv    := init.linv :: init.strides :: init.rp :: [init.inc];

        for p in vars.rp do
            Add(incr, assign(p, add(p, vars.inc)));
        od;

        self._offsetsInit(vars, init, cfg.coeff);
    end,

    _processPointer := meth(self, ptr, loop, vars, init, incr, map)
        local ld;
        # process pointers with separable base and loop dependent linear offset
        ld := Filtered(ptr.cmn, e -> self._contains(loop.var, e));
        if ld<>[]
           and ForAll(ld, s -> self._linear_exp(s, loop.var))
           and ForAll(ptr.offs, e -> not self._contains(loop.var, e.tail))
        then
            self._splitPtr(ptr, loop, vars, init, incr, map);
        fi;
    end,

    inject := (self, code, vars, prologue, epilogue) >> Cond(
         code _is decl,  decl(code.vars, self.inject(code.cmd, vars, prologue, epilogue)),
         code _is data,  data(code.var, code.value, self.inject(code.cmd, vars, prologue, epilogue)),
         # else
             decl(vars, chain(prologue, code, epilogue))),

    transformLoop := meth(self, l)
        local new_loop, ptrs, field, vars, init, incr, map, idx, oid, cmd, cp, subst;

        # collect references grouped by base pointers and extracted common summands
        ptrs := self._collectPtrs(l);

        # variables updated by _processPointer method
        #   vars.linv - loop invariant (outer) variables list
        #   vars.loop - inner variables list (declared in the loop body)
        #
        # initialization:
        #   init.linv - outer commands
        #   init.loop - inner commands
        #
        # incr - list of commands that increment pointers (loop body footer)
        # map - list of mapping records:
        #   map[i].ref - expression to replace
        #   map[i].subst
        #
        vars := rec( linv := [], loop := [] );
        init := rec( linv := [], loop := [] );
        incr := [];
        map  := [];

        for field in UserNSFields(ptrs) do
            self._processPointer( ptrs.(field), l, vars, init, incr, map );
        od;

        if map<>[] then
            # copy propagation of simple loop invariant expressions
            cp  := tab();
            init.linv := Filtered(init.linv, function(e) local prop;
                    prop := CondPat( e,
                              [assign, var, Value], true,
                              [assign, var, param], true,
                              false);
                    if prop then
                        cp.(e.loc.id) := e.exp;
                    fi;
                    return not prop;
                end);
            vars.linv := Filtered(vars.linv, e -> not IsBound(cp.(e.id)));

            subst := (c) -> SubstVars(c, cp);

            idx := Set(List(map, e -> e.ref));
            oid := Set(List(map, e -> ObjId(e.ref)));
            cmd := SubstTopDownNR_named(l.cmd, @(1, oid, x -> x in idx),
                    e -> subst(map[PositionProperty( map, x -> x.ref=e )].subst), "toRollingPointer");
            new_loop := decl( vars.linv, chain( subst(init.linv) :: [
                                loop( l.var, l.range, self.inject(cmd, subst(vars.loop), subst(init.loop), subst(incr)))
                        ]));
        else
            new_loop := l;
        fi;

        return new_loop;
    end,

    apply := meth(self, code)

        # normalize linear expressions, group summands and pull out loop variables
        code := SubstTopDownNR_named(code, @(1, self.opts.simpIndicesInside), e ->
                   SubstBottomUp(GroupSummandsExp(ESReduce(e, self.opts)), add, a->__groupSummandsVar(a.args)), "rollingPtrNormalize");

        # transform loops, innermost first
        code := SubstBottomUp(code, @(1, [loop, loopn]), e -> self.transformLoop(e));

        return code;
    end,
));


# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

########################################################################
#   vector constructs
########################################################################

IsSIMD_ISA := s -> IsRec(s) and IsBound(s.isSIMD_ISA) and s.isSIMD_ISA;
IsISA      := x -> IsRec(x) and IsBound(x.isISA) and x.isISA;

Class(AVecReg, AGenericTag, rec(
    isReg := true,
    isRegCx := false,
    isVec := true,
    updateParams := meth(self)
        Checked(IsSIMD_ISA(self.params[1]));
        Checked(Length(self.params)=1);
        self.v := self.params[1].v;
        self.isa := self.params[1];
    end,
    container := (self, spl) >> paradigms.vector.sigmaspl.VContainer(spl, self.isa)
));

Class(AVecRegCx, AVecReg, rec(
    updateParams := meth(self)
        Checked(IsSIMD_ISA(self.params[1]));
        Checked(Length(self.params)=1);
        self.v := self.params[1].v/2;
        self.isa := self.params[1];
    end,
    container := (self, spl) >> paradigms.vector.sigmaspl.VContainer(spl, self.isa.cplx()),
    isRegCx := true
));

# AMultiVec - list of ISAs, must be list of AVecReg tags in the future
#

Class(AMultiVec, AGenericTag, rec(
    isVec := true,
    updateParams := meth(self)
        Checked(ForAll(self.params, IsSIMD_ISA));
        Checked(Length(self.params)>=1);
    end,
));

Class(AISA, AGenericTag, rec(
    updateParams := meth(self)
        Checked(IsISA(self.params[1]));
        Checked(Length(self.params)=1);
        self.isa := self.params[1];
    end,
    # it's not a vectorized code, maybe defferent kind of containers?
    # containers do not go along well with OL though
    container := (self, spl) >> paradigms.vector.sigmaspl.VContainer(spl, self.isa)
));


# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

#F Replaces RemoteData inside a BB with RemoteDataUnrolled
UnrollRemoteData := function(e, cx)
   local extvars, loopvar, i, unrolledLoopVars, BBLoopVars, unrolledRange, newobj, newvar, newFofs, newofs, newfdataofs, f;

   # Compute list of loopvars that will be unrolled (ones inside BB)
   # These are all loops under the BB
   i := Length(cx.parents);
   unrolledLoopVars := [];
   unrolledRange := 1;
   while (ObjId(cx.parents[i])<>MultiBufISum and ObjId(cx.parents[i]) <> MultiBufISumFinal) do
      i := i-1;
      if ObjId(cx.parents[i])=ISum then
         unrolledLoopVars := unrolledLoopVars :: [cx.parents[i].var];
      fi;
   od;

   #Error("BP");

  # Take intersection because twiddle might not involve all loops under BB.
  BBLoopVars := Intersection(e.ofs.free(), unrolledLoopVars);
  unrolledRange := ListProduct(List(unrolledLoopVars, i->i.range));
  unrolledRange := When(unrolledRange=0, 1, unrolledRange);

  #NOTE: why was this an issue?
  #if BBLoopVars = [] then
  #   PrintLine("------------------> Going ahead with no Isum inside BB");
  #fi;

  # a) return RemoteDataUnrolled
  # b) with FDataOfs_mbuf that has a new range

  #Error("BP");

  newobj := e.child(1);    # This is the VRCDiag or similar obj

  # Change the FDataOfs_mbuf to include a different length and offset

  # Offset for FDataOfs_mbuf is RemoteData.ofs, with freevars that are not inside the BB set to zero.
  newFofs := Copy(e.ofs);

  extvars := newFofs.free();
  SubtractSet(extvars, BBLoopVars);

  for loopvar in extvars do
     SubstVars(newFofs, rec( (loopvar.id) := 0));
  od;

  newobj := SubstTopDown(newobj, FDataOfs_mbuf, f->FDataOfs_mbuf(f.var, f.len*unrolledRange, newFofs));

  e.var.t.size    := e.var.t.size    * unrolledRange;
  e.altbuf.t.size := e.altbuf.t.size * unrolledRange;

  # Must manually set unrolled loopvars to zero here because this will go to
  # init, and won't be taken care of by unrolling
  newofs := e.ofs;
  for loopvar in BBLoopVars do
     SubstVars(newofs, rec( (loopvar.id) := V(0) ));
  od;

  return(RemoteDataUnrolled(e.var, e.altbuf, e.value, e.ofs, newobj));
end;

Class(MBufCodegen, DistCodegen, rec(

    RemoteDataInit := meth(self, o, y, x, opts)
    #-----------------------------------------------------------------------------
        local r;
        r := Collect(o, RemoteDataUnrolled);
        # NOTE: handle multiple RemoteDataUnrolleds here.

        if Length(r) > 1 then
           Error("ERROR: Multi RemoteDataInit case NOT implemented!");
        fi;
        r := r[1];
        return(chain(
          self( RemoteDataNoBody(r.var, r.value, r.ofs, r.child(1)), y, x, opts ),
          self(o.child(1), y, x, opts)
        ));
    end,

    # This is called only from the BB body. So codegen only o.child(1)
    RemoteDataUnrolled := (self, o, y, x, opts) >> self( o.child(1), y, x, opts ),
    #-----------------------------------------------------------------------------

    # This is called in situations where there is no unrolled code.
    RemoteDataNoBody := meth(self, o, y, x, opts)
    #-----------------------------------------------------------------------------

      local pkSizeInBytes, dmaCommand;

      dmaCommand := var("GATHMEM_DIAG");
      pkSizeInBytes := o.child(1).element.len * opts.vector.vlen * (opts.vector.isa.bits/8);

      if pkSizeInBytes > 16384 then
         if pkSizeInBytes = 32768 then
            dmaCommand := var("GATHMEM_DIAG_32K");
         elif pkSizeInBytes = 65536 then
            dmaCommand := var("GATHMEM_DIAG_64K");
         else
           Error("Too large a packet size for getting twiddles");
         fi;
      fi;
     return(chain(
       call(dmaCommand,
       o.altbuf.id,    # Destination base address
       0,              # Destination offset
       "spe_info."::o.value.id,     # Source base address
       o.ofs*opts.vector.vlen,       # Source offset
       o.child(1).element.len * opts.vector.vlen # Packet size in elements
       )

       # Only if

       #call(var("spu_writech"),
       #"MFC_WrTagMask",
       #"1 << 3"),
       #
       #call(var("spu_mfcstat"),
       #"MFC_TAG_UPDATE_ALL"
       #)
     ));
    end,

    # # These should've gotten converted to RemoteDataUnrolled and RemoteDataInit
    # RemoteData := meth(self, o, y, x, opts)
    #     Error("# These should've gotten converted to RemoteDataUnrolled and RemoteDataInit");
    # end,

    # These should've gotten converted to RemoteDataUnrolled and RemoteDataInit
    # But during DP, these might not always be contained in a MultiBufISum. In
    # this case, we should ideally add a new MultiBufISum. Here's a hack instead:
    #RemoteData := (self, o, y, x, opts) >> self(o.child(1), y, x, opts),
    RemoteData := (self, o, y, x, opts) >> skip(),

#-----------------------------------------------------------------------------
    Formula := meth(self, o, y, x, opts)
#-----------------------------------------------------------------------------
#NOTE: Formula needs major cleanup Tue 31 Mar 2009 12:58:40 AM EDT
        local icode, datas, datas_ppe, prog, params, sub, initsub, initsub_ppe, io, e, d;

        [x, y] := self.initXY(x, y, opts);

        o :=  Process_fPrecompute(o, opts);

        o := o.child(1);
        params := Set(Concatenation(Collect(o, param), Filtered(Collect(o, var), IsParallelLoopIndex)));

        SubstTopDown(o, @@(1, VRCDiag,
            (e,cx)->(ObjId(e.element)=FDataOfs
                     and e.element.var.t.size * opts.vector.vlen >= opts.maxSPETwiddles
                     #and IsBound(cx.MultiBufISum)
                     #and IsBound(cx.BB)
                     )
            ),
          e->let(v := @@(1).val,
                 f     := v.element,
                 dmbuf     := var.fresh_t("Tmbuf",  TArray(f.var.t.t, f.len)),
                 dmbuf_alt := var.fresh_t("TmbufAlt", TArray(f.var.t.t, f.len)),
                 RemoteData(dmbuf, dmbuf_alt, f.var, f.ofs, VRCDiag(FDataOfs_mbuf(dmbuf, f.len, 0), v.v))
              )
        );

        # Replace FDataOfs that won't fit inside an SPE with FDataOfs_mbuf
        #SubstTopDown(o, @@(1, FDataOfs,
        #    (e,cx)->(e.var.t.size*opts.vector.vlen >= opts.maxSPETwiddles
        #             #and IsBound(cx.MultiBufISum)
        #             #and IsBound(cx.BB)
        #             )
        #    ),
        #e->let(f:=@@(1).val, FDataOfs_mbuf(f.rChildren()[1], f.rChildren()[2], f.rChildren()[3]))
        #);

        datas := Collect(o, FDataOfs);

        datas_ppe := Collect(o, RemoteData);

        #datas_ppe := [];
        #for d in datas do
        #   if d.var.t.size * opts.vector.vlen >= opts.maxSPETwiddles then
        #      datas_ppe := datas_ppe :: [d];
        #      datas := RemoveList(datas, d);
        #   fi;
        #od;

        # Replace RemoteData with RemoteDataUnroll where necessary
        #NOTE: Checking for context is not enough: must check for length
        #SubstTopDown(o, @@(1, RemoteData, (e,cx)->IsBound(cx.BB)), UnrollRemoteData);

        #SubstTopDown(o, @@(1, RemoteData, (e,cx)->(IsBound(cx.MultiBufISum) or IsBound(cx.MultiBufISumFinal))),
        #    UnrollRemoteData);

        SubstTopDown(o, @@(1, RemoteData, (e,cx)->IsBound(cx.MultiBufISum) or IsBound(cx.MultiBufISumFinal)),
            UnrollRemoteData);

        # Enclose BBs within a RemoteDataInit if needed (if the BB contains one or more
        # RemoteDataUnrolled's)
        #SubstTopDown(o, @@(1, MultiBufISum, (e,cx)->(
        #    Length(Collect(e, RemoteDataUnrolled))>=1
        #    and (not IsBound(cx.RemoteDataInit) or Length(cx.RemoteDataInit) = 0)
        #  )),
        #  f->RemoteDataInit(f)
        #);

        #return(o);

        o := BlockSumsOpts(o, opts);
        icode := self(o, y, x, opts);
        icode := ESReduce(icode, opts);
        icode := RemoveAssignAcc(icode);
        icode := BlockUnroll(icode, opts);
        # icode := PowerOpt(icode);
        icode := DeclareHidden(icode);
        if IsBound(opts.isFixedPoint) and opts.isFixedPoint then
            icode := FixedPointCode(icode, opts.bits, opts.fracbits);
        fi;

        io := When(x=y, [x], [y, x]);
        sub := Cond(IsBound(opts.subName), opts.subName, "transform");
        initsub := Cond(IsBound(opts.subName), Concat("init_", opts.subName), "init");
        initsub_ppe := Cond(IsBound(opts.subName), Concat("init_ppe_", opts.subName), "init_ppe");
        icode := func(TVoid, sub, Concatenation(io, params), icode);

        #Error("BP");

        if IsBound(opts.generateInitFunc) and opts.generateInitFunc then
            prog := program(
                #decl(List(datas, x->x.var)::List(datas_ppe, x->x.var),
                decl(List(datas, x->x.var),
                    chain(
                        func(    TVoid, initsub,     params, chain(List(datas,     x -> SReduce(x.var.init, opts)))),
                        func_ppe(TVoid, initsub_ppe, params, decl(List(datas_ppe, x->x.value), chain(List(datas_ppe, x -> SReduce(x.value.init, opts)))) ),
                        icode
                    )));
        else
            prog := program( func(TVoid, initsub, params, chain()), icode);
        fi;

        # FF: I really don't know why suddenly AVX_8x32f requires me to do that !!
        if IsBound(opts.vector.isa.fixProblems) then prog := opts.vector.isa.fixProblems(prog, opts); fi;

        prog.dimensions := o.dims();

        # Perform software pipelining
        # prog := MarkPreds(prog);
        # prog := MarkDefUse(prog);
        # SubstTopDown(prog, @(1, loop, e->(Length(Collect(e, loop))=1 and Length(e.range) >= 4)),
        #     e->let(l := @(1).val, loop_sw(l.rChildren()[1], l.rChildren()[2], l.rChildren()[3]) )
        # );
        # SubstTopDown(prog, loop_sw, e->SoftwarePipeline(e));

        return prog;
    end,

    GathMem := meth(self, o, y, x, opts)
        #Gath: loop(i, o.func.domain(), assign(nth(y,i), nth(x, func.at(i))));
        #DO_DMA(Xalt, spe_info.X, IDEAL_DMA_SIZE_BYTES, MFC_GET_CMD); // Xalt = spe_info.X
        local func, pkSize, i, numPkts, pkSizeInBytes, dmaCommand;
        func        := o.func.lambda();
        pkSize      := o.pkSize;
        numPkts     := o.func.domain();
        #i          := 0;
        i           := Ind(numPkts);

        dmaCommand := var("GATHMEM_GET");
        pkSizeInBytes := o.pkSize * (opts.vector.isa.bits/8);

        #NOTE: 16384 is the largest packet we can send on the Cell
        # If we're not using tables, then we can go higher using backend macros
        if (pkSizeInBytes > 16384) then
          # NOTE: How to handle cases where pkSize is greater than 16k?
          if (pkSizeInBytes = 32768) then
             dmaCommand := var("GATHMEM_GET_32K");
          elif (pkSizeInBytes = 65536) then
             dmaCommand := var("GATHMEM_GET_64K");
          elif (pkSizeInBytes = 131072) then
             dmaCommand := var("GATHMEM_GET_128K");
          else
          Error("Selected packet size - ", o.pkSize, " - is higher than the Cell's max allowed (using macros).");
          fi;
       fi;

        # NOTE: remove this after testing
        if (numPkts <= 16 ) then
            #16 is the size of the SPU's DMA Command queue. Exceeding this
            #means taking a huge performance loss because in effect, DMAs won't
            #be done in the background.
            # NOTE: this (16) is hardcoded for the implementation than for the ISA
            return(loop(i, numPkts, call(
                   dmaCommand,
                   #y,
                   var(Concatenation(y.id, "alt")),
                   i*pkSize,
                   var(Concatenation("spe_info.", y.id)),
                   func.at(i)*pkSize,
                   pkSize
                )));
        fi;

        if (numPkts > 2048) then
          Error("Selected attemping to DMA more than 2048 packets at one time.  The Cell architecture has no facility for this. (Doing multiple sets is possible, but in most cases, a better algorithm is needed).");
        fi;

        #NOTE: 16384 is the largest packet we can send on the Cell using DMA tables
        if (pkSizeInBytes > 16384) then
          Error("Selected packet size - ", o.pkSize, " - is higher than the Cell's max allowed.");
        fi;

        return(chain(
            # Build DMA list
            loop(i, numPkts, chain(
              call(var("GATH_LIST_SETSIZE"), pkSize, i.id),
              call(var("GATH_LIST_SETADDR"), func.at(i)*pkSize, i.id, var(Concatenation("spe_info.", y.id)))
            )),

            # Execute DMA list
            call(var("GATHMEM_LIST"),
                var(Concatenation(y.id, "alt")),
                var(Concatenation("spe_info.", y.id)),
                numPkts
            )
        ));

    end,

    ScatMem := meth(self, o, y, x, opts)
        #Scat: return loop(i, o.func.domain(), assign(nth(y,func.at(i)), nth(x, i)));
        local func, pkSize, i, numPkts, pkSizeInBytes, dmaCommand;
        func          := o.func.lambda();
        pkSize        := o.pkSize;
        numPkts     := o.func.domain();
        #i          := 0;
        i           := Ind(numPkts);

        pkSizeInBytes := o.pkSize * (opts.vector.isa.bits/8);

        #NOTE: 16384 is the largest packet we can send on the Cell
        # If we're not using tables, then we can go higher using backend macros
        dmaCommand := var("SCATMEM_PUT");
        pkSizeInBytes := o.pkSize * (opts.vector.isa.bits/8);

        #NOTE: 16384 is the largest packet we can send on the Cell
        # If we're not using tables, then we can go higher using backend macros
        if (pkSizeInBytes > 16384) then
          # NOTE: How to handle cases where pkSize is greater than 16k?

          if (pkSizeInBytes = 32768) then
             dmaCommand := var("SCATMEM_PUT_32K");
          elif (pkSizeInBytes = 65536) then
             dmaCommand := var("SCATMEM_PUT_64K");
          elif (pkSizeInBytes = 131072) then
             dmaCommand := var("SCATMEM_PUT_128K");
          else
          Error("Selected packet size - ", o.pkSize, " - is higher than the Cell's max allowed (using macros).");
          fi;
        fi;

        #DO_DMA(Yalt, spe_info.Y, IDEAL_DMA_SIZE_BYTES, MFC_PUT_CMD); // spe_info.Y = Yalt

        if (numPkts <= 16 ) then
            return(loop(i, numPkts, call(
                dmaCommand,
                #x,
                var(Concatenation(x.id, "alt")),
                i*pkSize,
                var(Concatenation("spe_info.", x.id)),
                func.at(i)*pkSize,
                pkSize
            )));
        fi;

        if (numPkts > 2048) then
          Error("Selected attemping to DMA more than 2048 packets at one time. The Cell architecture has no facility for this");
        fi;

        #NOTE: 16384 is the largest packet we can send on the Cell using DMA tables
        if (pkSizeInBytes > 16384) then
          Error("Selected packet size - ", o.pkSize, " - is higher than the Cell's max allowed for DMA lists.");
        fi;

        return(chain(
            # Build DMA list
            loop(i, numPkts, chain(
              call(var("SCAT_LIST_SETSIZE"), pkSize, i.id),
              call(var("SCAT_LIST_SETADDR"), func.at(i)*pkSize, i.id, var(Concatenation("spe_info.", x.id)))
            )),

            # Execute DMA list
            call(var("SCATMEM_LIST"),
                var(Concatenation(x.id, "alt")),
                var(Concatenation("spe_info.", x.id)),
                numPkts
            )
        ));

    end,

     R2Sum := (self, o, y, x, opts) >>
            loop(o.var1, o.domain1, loop(o.var2, o.domain2, self(o.child(1), y, x, opts))),

    # Note: to perform multibuffering, x,y are reversed in some of the following calls.
    MultiBufISum := meth(self, o, y, x, opts)
       local r, mbufvars, mloop, loopbody;

        r := Collect(o, RemoteDataUnrolled);

        # NOTE: handle multiple RemoteDataUnrolleds here.
        if Length(r) > 1 then Error("ERROR: Multi RemoteDataInit case NOT implemented!"); fi;

        mbufvars := [];
        if Length(r) > 0 then
           r := r[1];
           mbufvars := [r.var]::[r.altbuf];
        fi;

        mloop := multibuffer_loop;
        if IsBound(opts.nombuf) and opts.nombuf=true then mloop := mem_loop; fi;

#loopvar, range, y, x, gathmem, twiddles, bufs, cmd, scatmem)
        return(decl(mbufvars,
          mloop(o.var, o.domain, y, x,
               self(o.gathmem, x, y, opts),
               When(r=[], [], self( RemoteDataNoBody(r.var, r.altbuf, r.value, r.ofs, r.child(1)), y, x, opts )),
               mbufvars,
               self(o.child(1), y, x, opts),
               self(o.scatmem, x, y, opts)
          )
        ));
    end,

    MultiBufISumFinal := (self, o, y, x, opts) >>
        self(MultiBufISum(o.var, o.domain, o.scatmem, o.child(1), o.gathmem), y, x, opts),

    #HACK: this won't work if twiddles need to be streamed.
    MemISum := (self, o, y, x, opts) >>
       mem_loop(o.var, o.domain, y, x,
            self(o.gathmem, x, y, opts),
            [],
            [],
            self(o.child(1), y, x, opts),
            self(o.scatmem, x, y, opts)
       ),

    MemISumFinal := (self, o, y, x, opts) >>
       mem_loop(o.var, o.domain, y, x,
            self(o.gathmem, x, y, opts),
            self(o.child(1), y, x, opts),
            self(o.scatmem, x, y, opts)
       ),

    MultiBufDistSum := (self, o, y, x, opts) >>
        self(MultiBufISum(o.var, o.domain, o.scatmem, o.child(1), o.gathmem), y, x, opts)
));


##  Copyright (c) 2018-2021, Carnegie Mellon University
##  See LICENSE for details

_dim3 := d -> "dim3("::d.id::")";

Class(HIPUnparser, CudaUnparser, rec(
    cu_call := (self, o, i, is) >>
        Print(Blanks(i), "hipLaunchKernelGGL(",
                self.infix([o.func, _dim3(o.dim_grid), _dim3(o.dim_block), "0", "0"]::o.args, ", "), ");\n"),
));


# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

neg.doPeel := true;

#F CopyPropagate(<code>)
#F     Performs copy propagation and strength reduction
#F     If <code> has IFs it MUST be in SSA form, i.e.,
#F     SSA(<code>) must be run first.
#F
Class(SubstVarsRules, RuleSet, rec(
    create := (self, varmap) >> Inherit(self,
	rec(
            rules := rec(
		v := Rule(@(1,var,e->IsBound(varmap.(e.id))), e -> varmap.(e.id))
	    )
	)).compile()
));

# apply linear normalization to index expressions, this will simplify cases like
# 3x + 2x = 5x
#
Class(SimpIndicesRules, RuleSet, rec(
    create := (self, opts) >> Inherit(self,
	rec(
	    rules := rec(
		simp_indices := Rule(@(1, opts.simpIndicesInside), x -> GroupSummandsExp(x))
	    )
	)).compile()
));

# Transforms nth(x, idx) to deref(x + idx)
Class(RulesDerefNth, RuleSet);
RewriteRules(RulesDerefNth, rec(
    deref_nth := Rule(
	[nth, @(1).cond( x -> not (x _is [Value, param]) or not IsBound(x.value)), @(2)],
	e -> let(
	    b := @(1).val, idx := @(2).val,
	    Cond(
		ObjId(idx) = add, deref(ApplyFunc(add, [b] :: idx.args)),
		ObjId(idx) = sub, deref(ApplyFunc(add, [b] :: [idx.args[1], neg(idx.args[2])])),
                                  deref(b + idx))))
));

# sorts the incoming array of assignments by the offset of the
# variable being assigned TO, rather than the one assigned FROM.
# improves locality hence cache performance.
_sortByIdx := function(array)
    local newarray;
    newarray := Copy(array);

    SortParallel(
        List(array, e -> Double(SubString(e.args[2].id, 2))),
        newarray
    );

    return newarray;
end;

CopyTab := function (t)
   local r, a;
   r := tab();
   for a in NSFields(t) do
      r.(a):=Copy(t.(a));
   od;
   return r;
end;

Class(CopyPropagate, CSE, rec(
    propagate := (self, cloc, cexp) >> let(cebase := ObjId(cexp),
        ((cebase in [var, Value, noneExp])
            or (self.propagateNth and cebase in [nth, deref] and IsValue(cexp.idx))
            or (IsBound(self.opts.autoinline) and IsBound(cloc.succ) and Length(cloc.succ)<=1)
            or (IsBound(cloc.succ) and Length(cloc.succ)=0))),

    # sreduce_and_subst(c.exp) can lead to inf loop - why?
    procRHS := (self, x) >> self.sreduce(self.subst3(self.sreduce(self.subst2(x)))),

    procLHS := (self, x) >> self.sreduce3(self.sreduce(self.subst2(x))),

    # NOTE: explain this
    prepVarMap := meth(self, initial_map, do_sreduce, do_init_scalarized, do_idx)
        local v, varmap, rs_subst, rs2, rs_deref, rs_idx;
        self.varmap := initial_map;

        if do_sreduce then
	    # do_idx is done initially, at the same time as scalarization
	    # deref prevents scalarization, so  must be disabled at that moment
	    rs_deref := When(not do_idx and self.opts.useDeref, RulesDerefNth, EmptyRuleSet);
	    rs_idx   := When(do_idx, SimpIndicesRules.create(self.opts), EmptyRuleSet);

            rs_subst := SubstVarsRules.create(self.varmap);
	    rs2 := MergedRuleSet(rs_subst, rs_deref);
	    rs_subst.__avoid__ := [Value];
	    rs_idx.__avoid__ := [Value];
	    rs2.__avoid__ := [Value];

            self.subst   := x -> SubstVars(x, self.varmap);
            self.subst2  := x -> BU(x, rs2);
            self.subst3  := x -> SubstBottomUpRules(BU(x, rs_subst), rs_idx.rules);
            self.sreduce := x -> SReduce(x, self.opts);
            self.sreduce3 := x -> SubstBottomUpRules(x, rs_idx.rules);
            self.sreduce_and_subst := MergedRuleSet(rs_subst, rs_deref, RulesStrengthReduce);
        else
            self.subst   := x -> SubstVars(x, self.varmap);
            self.subst2  := x -> SubstVars(x, self.varmap);
            self.subst3  := x -> SubstVars(x, self.varmap);
            self.sreduce := x -> x;
            self.sreduce_and_subst := self.subst;
        fi;

        if do_init_scalarized then
            for v in compiler.Compile.scalarized do
                self.varmap.(v.id) := noneExp(v.t);
            od;
        fi;
        return self;
    end,

    assumeSSA := false,
    afterSSA := self >> WithBases(self, rec(assumeSSA:=true)),

    closeVarMap := meth(self, other, newcmds)
        local v;
        for v in UserNSFields(other.varmap) do
            if (not IsBound(self.varmap.(v)) or self.varmap.(v) <> other.varmap.(v)) and SuccLoc(var(v))<>[] then
                if self.assumeSSA then ;
                #    self.varmap.(v) := other.subst(other.varmap.(v));
                #    PrintLine("close ", v, " => ", other.subst(other.varmap.(v)));
                else
                    Unbind(self.varmap.(v));
                    Add(newcmds.cmds, assign(var(v), other.subst(other.varmap.(v))));
                fi;
            fi;
        od;
        #Print("-----\n");
    end,

    init := meth(self, opts)
        self.opts := opts;
        self.prepVarMap(tab(), true, true, false);
        self.doScalarReplacement := opts.doScalarReplacement;
        self.propagateNth := opts.propagateNth;
        self.flush();
        return self;
    end,

    initial := meth(self, code, opts)
        self.opts := opts;
        self.prepVarMap(tab(), true, true, true);
        self.doScalarReplacement := opts.doScalarReplacement;
        self.propagateNth := opts.propagateNth;
        self.flush();
        return self.copyProp(code);
    end,

    __call__ := (self, code, opts) >> self.init(opts).copyProp(code),

    flush := meth(self)
         self.csetab := tab();
         if (self.doScalarReplacement) then self.lhsCSE := CSE.init(); fi;
    end,

    fast := meth(self, code, opts)
        self.prepVarMap(tab(), false, false, false);
        self.doScalarReplacement := opts.doScalarReplacement;
        self.propagateNth := opts.propagateNth;
        self.opts := opts;
        self.flush();
        return self.copyProp(code);
    end,

    procAssign := meth(self, c, newcmds, live_out)
        local newloc, newexp, cid, cloc, op, varmap, lkup;
        varmap := self.varmap;

        # NOTE: generalize this, use .in()/.out()/.inout() somehow
        if IsBound(c.p) then
            cid := (loc, exp) -> ObjId(c)(loc, exp, c.p);
        else cid := ObjId(c); fi;

        # to my current understanding, if we assign a phi function, nothing can be propagated.
        # If you propagate inside a phi, you lose the branch information.
        if (ObjId(c.exp)=phi) then Add(newcmds, c); return; fi;

        # Run strength reduction, variable remapping, and all other rewrite rules
        newexp := self.procRHS(c.exp);
        cloc := When(not IsVar(c.loc) or (c.loc in c.op_inout()),
	             self.procLHS(c.loc), c.loc);

        # check if newexp was already computed
        lkup := self.cseLookup(newexp);
        if lkup <> false then newexp := lkup; fi;

	# if cloc is marked as live_out, save it in the list, so that its not kicked out
        When(IsVar(cloc) and IsBound(cloc.live_out), AddSet(live_out, cloc));

        # invalidate the LHS in the CSE tables
        if (self.cseLookup(cloc) <> false)
            then self.cseInvalidate(cloc); fi;
        if (self.doScalarReplacement and self.lhsCSE.cseLookup(cloc) <> false)
            then self.lhsCSE.cseInvalidate(cloc); fi;

        # propagate
        if IsVar(cloc) and cid=assign and self.propagate(cloc,newexp) then
            # this should not happen due to sreduce/subst above
            When(IsVar(newexp) and IsBound(varmap.(newexp.id)), Error("Should not happen"));
            varmap.(cloc.id) := newexp;

        # do not propagate
        else
            # NOTE: YSV: is this right???
            #        it seems that its correct, above case catches var=noneExp,
            #        so this looks like a mem-store, e.g.,  nth(..) = noneExp
            if ObjId(newexp)=noneExp then return; fi;

            # do not propagate AND cloc is a variable
            if IsVar(cloc) and not (cloc in c.op_inout()) then
                # propagate 'neg' like unary operators outwards
                if IsBound(newexp.doPeel) and newexp.doPeel then
                    op := ObjId(newexp);
                    newexp := newexp.args[1];
                    if self.propagate(cloc, newexp) then
                        if IsVar(newexp) and IsBound(varmap.(newexp.id)) then
                            varmap.(cloc.id) := op(varmap.(newexp.id));
                        else
                            varmap.(cloc.id) := op(newexp);
                        fi;
                    else
                        newloc := cloc.clone();
                        varmap.(cloc.id) := op(newloc);
                        Add(newcmds, cid(newloc, newexp));
                        # careful! cid can be any storeop, not always <assign>
                        When(cid=assign, self.cseAdd(newloc, newexp));
                    fi;
                else
                    # variable that is not propagated is remapped to a fresh name (to get code in SSA form)
                    newloc := cloc.clone();
                    varmap.(cloc.id) := newloc;
                    Add(newcmds, cid(newloc, newexp));   # cid == assign | assign_nop | ...
                    # careful! cid can be any storeop, not always <assign>
                    When(cid=assign, self.cseAdd(newloc, newexp));
                fi;

            #  do not propagate AND cloc is *not* a variable (ie. nth(X, i)) or it is an inout variable
            else
                if self.doScalarReplacement or not (self.propagateNth or IsVar(newexp) or IsValue(newexp)) then
                    # for non-variable newexp a fresh temporary var sXX is created to hold the result
                    newloc := var.fresh_t("s", newexp.t);
                    self.cseAdd(newloc, newexp);
                    Add(newcmds, assign(newloc, newexp));
                    newexp := newloc;
                fi;

                if self.doScalarReplacement then
                    When(cid<>assign or not (ObjId(cloc) in [nth,deref]),
                        Error("Scalar replacement can't handle <cloc> of type ", ObjId(cloc)));
                    self.cseAdd(newexp, cloc);
                    self.lhsCSE.cseAdd(newexp, cloc);
                else
                    Add(newcmds, cid(cloc, newexp));
                fi;
            fi;
        fi;
    end,

    procIF := meth(self, c, newcmds, live_out)
        local lo_map, v, orig, then_cmd, else_cmd, then_cp, else_cp;
        c.cond := self.procRHS(c.cond);

        if IsValue(c.cond) then
            Error("This is a constant IF, it should be folded away before reaching copyprop");
            # Following code should work if needed (if you uncomment the error) but it is not optimized
            # self.flush(); # YSV: why is this needed??? I will comment this out.
            if c.cond.v=0 or c.cond.v=false then  Add(newcmds, self.copyProp(c.else_cmd));
            else                                  Add(newcmds, self.copyProp(c.then_cmd));
            fi;
        else
            # Here the idea is that each part of the branch should work with
            # the original csetab (there cannot be crossings) and the final thing
            # should be restored to the original csetab

            then_cp := CopyFields(self, rec(csetab := CopyTab(self.csetab)))
                .prepVarMap(CopyTab(self.varmap), true, false, false);

            else_cp := CopyFields(self, rec(csetab := CopyTab(self.csetab)))
                .prepVarMap(CopyTab(self.varmap), true, false, false);

            then_cmd := then_cp.copyProp(c.then_cmd);
            self.closeVarMap(then_cp, then_cmd);

            else_cmd := else_cp.copyProp(c.else_cmd);
            self.closeVarMap(else_cp, else_cmd);

            Add(newcmds, IF(c.cond, then_cmd, else_cmd));
            #self.flush();
        fi;
    end,

    #if we do Scalar Replacement on the fly, reinject the final writes as assigns (for now)
    finalizeScalarReplacement := meth(self, newcmds)
        local entry;
            if IsBound(self.lhsCSE.csetab.nth) then
                # MRT: sort by order of variable being assigned TO rather
                # than variable assigned FROM. This reduces cache misses by
                # exploiting possible locality in the output array.
                #
                # eg: Y[0]  := a1   ===>  Y[0]   := a1
                #     Y[32] := a2         Y[1]   := a3
                #     Y[1]  := a3         Y[32]  := a2

                for entry in _sortByIdx(self.lhsCSE.csetab.nth) do
                    if Length(entry.args)=2 then
                        Add(newcmds,
                            assign(nth(entry.args[1],entry.args[2]),
                                entry.loc));
                    fi;
                od;
            fi;
            if IsBound(self.lhsCSE.csetab.deref) then
                for entry in self.lhsCSE.csetab.deref do
                    if Length(entry.args)=1 then
                        Add(newcmds,
                            assign(deref(entry.args[1]),
                        entry.loc));
                    fi;
                od;
            fi;
    end,

    # create the reverse mapping, to map last assignment to live_out variable
    #  to the actual variable intead of an SSA related substitute
    substLiveOut := meth(self, newcmds, live_out)
        local varmap, lo_map, active, v;
        varmap := self.varmap;
        lo_map := tab();
        active := Set(Collect(newcmds, var));  # explain this
        for v in live_out do
           # NOTE: The commented out lines with #X removed extra copies
           #  associated with live_out variables. But it turns out that this
           #  does not work if a live_out variable is never actually remapped,
           #  but has an entry in varmap due to copy propagation.
           #  For example (a = t11; b = a) if a is live_out will be compiled to
           #  (b=t11) b CopyPropagate and converted to (b=a) by function below.
           #   which is incorrect

           #X if not IsVar(varmap.(v.id)) or not (varmap.(v.id) in active) then
                Add(newcmds, assign(v, varmap.(v.id)));
           #X else
           #X     lo_map.(varmap.(v.id).id) := v;
           #X fi;
            Unbind(varmap.(v.id));   # prevent duplication of cleanup code due to closeVarMap
        od;
        newcmds := SubstVars(newcmds, lo_map);
        return newcmds;
    end,

    copyProp := meth(self, code)
        local c, cid, cmds, newcmds, live_out;
        cmds := When(ObjId(code)=chain, code.cmds, [code]);
        newcmds := [];
        live_out := Set([]);
        for c in cmds do
            cid := ObjId(c);
            if   IsAssign(c)     then  self.procAssign(c, newcmds, live_out);
            elif (cid = IF)      then  self.procIF(c, newcmds, live_out);
            elif IsExpCommand(c) then  Add(newcmds, self.procRHS(c));
            elif (cid = skip)    then  ;  # do nothing

            # if .sideeffect is bound, the command is assumed have side effects
            # so one has to enable array scalarization inside all its children
            elif IsRec(c) and IsBound(c.sideeffect) and c.sideeffect then
               Add(newcmds, map_children_safe(c, x -> self.procRHS(x)));

            # the command is a container for other functions, copyprop all children
            else Add(newcmds, map_children_safe(c,
			x -> Cond(IsCommand(x), self.copyProp(x), self.procRHS(x))));
            fi;
        od;

        When(self.doScalarReplacement, self.finalizeScalarReplacement(newcmds));
        newcmds := self.substLiveOut(newcmds, live_out);
        # self.flush();
        return chain(newcmds);
    end
));


# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

memo := function(context, prefix, exp)
   local v, isum;
   exp := SReduce(toExpArg(exp), SpiralDefaults); ##Pass the real opts instead
   if IsValue(exp) then return exp.v;
   elif IsVar(exp) then return exp;
   else
       if IsBound(context.ISum) and context.ISum <> [] then
       isum := Last(context.ISum);
       v := var.fresh_t(prefix, TInt);
       if not IsBound(isum.memos) then isum.memos := []; fi;
       Add(isum.memos, [v, exp]);
       #Print(v.id , " -> ", exp, " [", isum.var, " in ", isum.domain, "]\n");
       v.mapping := exp;
       return v;
       else
       return exp;
       fi;
   fi;
end;

nomemo := (cx,pfx,exp) -> exp;

memo := nomemo; # memo feature is currently broken -- disable it

Class(ProcessMemos, RuleSet);
RewriteRules(ProcessMemos, rec(
   # pull out cmemos
   PullOutCmemo := Rule(cmemo,
     function(e,cx)
         local target;
	 target := cx.(e.target);
         if not IsBound(target.memos) then
	     target.memos := [ e.args[1], e.mapping ];
	 else Add(target.memos, [ e.args[1], e.mapping ]);
	 fi;
	 return e.args[1];
     end),

   # process memos at ISum
   ProcessISumMemos := Rule(@(1, [ISum, ISumAcc], e->IsBound(e.memos)),
     e -> ObjId(@(1).val)(e.var, e.domain,
	      FoldL(Reversed(e.memos),
		  (ex,m) -> Data(m[1], m[2], ex)), e.child(1)))
));


# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

########################################################
#   Machinery for being able to translate
#   vector-sigma-sums into vector code
########################################################

vref := (loc, idx, vlen) -> When(vlen=1,
    nth(loc, idx),
    nth(tcast(TPtr(TVect(loc.t.t, vlen)), loc), idx/vlen));

nth.toPtr := (self, t) >> let(
    exp := When(IsPtrT(self.loc.t), self.loc, tcast(self.loc.t.toPtrType(), self.loc)) + self.idx,
    exp_t := exp.t,
    new_t := When( IsPtrT(exp_t), TPtr(t, exp_t.qualifiers).withAlignment(exp_t), TPtr(t)),
    When(new_t=exp_t, exp, tcast(new_t, exp))
);

DeclareVars := function(code)
    local vars, vects,good;

    vars  := Set(List(Collect(code, [assign, @(1,var), @]),     e->e.loc));
    vects := Set(List(Collect(code, [assign, [vref,@(1,var),@], @]), e->e.loc.loc));
    good:=Set(Flat(List(Collect(code, decl), e->e.vars)));
    SubtractSet(vects, good);
    SubtractSet(vects, Set([X,Y]));
    SubtractSet(vars, good);

    if Length(vars) > 0 then code := decl(vars, code); fi;
    if Length(vects) > 0 then code := decl(vects, code); fi;
    return code;
end;


# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

Class(ScratchpadGlobals, rec(
    getOpts := meth(arg)
        local lssize, opts, swp, brules, nrules, br, nrsgmts, vlen, globalUnrolling,size, ttype;

	lssize := When (Length(arg) >= 2, arg[2], 2);
	nrsgmts := When (Length(arg) >= 3, arg[3], 1);
	vlen := When (Length(arg) >= 4, arg[4], 1);
	size := When (Length(arg) >= 5, arg[5], 2);
    ttype := When (Length(arg) >= 6, arg[6], 'R');
	swp := When (Length(arg) >= 7, arg[7], false);
	globalUnrolling := When(Length(arg) >=8, arg[8], 1);

    brules := When(IsRec(SpiralDefaults.breakdownRules),
        UserRecFields(SpiralDefaults.breakdownRules),
        Filtered(Dir(SpiralDefaults.breakdownRules), i->not i in SystemRecFields));
    nrules := rec();
    for br in brules do
        nrules.(br) := List(SpiralDefaults.breakdownRules.(br), i->CopyFields(i));
    od;
    opts := CopyFields(SpiralDefaults);
    opts.breakdownRules := nrules;

    opts.breakdownRules.TCompose := [ TCompose_tag];
    opts.breakdownRules.DFT := [DFT_Base, DFT_CT, DFT_tSPL_CT, DFT_PD, DFT_Rader];
    opts.breakdownRules.TTwiddle := [ TTwiddle_Tw1];

	opts.breakdownRules.WHT := [WHT_tSPL_BinSplit, WHT_Base, WHT_BinSplit];

	opts.breakdownRules.TTensor := [AxI_IxB,IxB_AxI];
	opts.breakdownRules.TTensorI := Concat([IxA_scratch_push, IxA_base, AxI_base, IxA_L_base, L_IxA_base], [IxA_scratch, AxI_scratch, IxAL_scratch]);

	opts.tags := [Cond( ttype = 'R', ALStore(lssize,nrsgmts,vlen), ALStoreCx(lssize,nrsgmts,vlen)) ];

	opts.formulaStrategies.sigmaSpl := [ MergedRuleSet(RulesSumsScratch, RulesFuncSimpScratch, RulesDiag, RulesDiagStandalone, RulesStrengthReduce, RulesRCScratch,RulesII,OLRules) ];
    opts.formulaStrategies.preRC := [ MergedRuleSet(RulesSumsScratch, RulesFuncSimpScratch, RulesDiag, RulesDiagStandalone, RulesStrengthReduce, RulesRCScratch, RulesII, OLRules), (s,o) -> ScratchModel.updateInfo(s) ];
	opts.formulaStrategies.rc := [ MergedRuleSet(RulesSums, RulesFuncSimp, RulesDiag, RulesDiagStandalone, RulesStrengthReduce, RulesRCScratch, RulesII, OLRules) ];
    #opts.formulaStrategies.postProcess := [(s, opts) -> compiler.BlockSums(opts.globalUnrolling, s)];
    opts.size := size;
	opts.swp := swp;
    opts.globalUnrolling := globalUnrolling;

    opts.sumsgen := ScratchSumsGen;
	opts.codegen := ScratchCodegen;
    opts.unparser := CScratchUnparserProg;

    opts.memModifier := "__memory";
    opts.scratchModifier := "__scratch";
    opts.arrayDataModifier := "__rom";
    opts.romModifier := "__rom";

	opts.includes := [];
    Add(opts.includes, "\"scratch.h\"");

    opts.dmaSignal := (self, opts) >> "DMA_signal";
    opts.dmaWait := (self, opts) >> "DMA_wait";
    opts.cpuSignal := (self, opts) >> "CPU_signal";
    opts.cpuWait := (self, opts) >> "CPU_wait";
    opts.dmaFence := (self, opts) >> "DMA_fence";
    opts.dmaLoad := (self, opts) >> "DMA_load";
    opts.dmaStore := (self, opts) >> "DMA_store";
    opts.model := ScratchModel;

    return opts;
    end
));


# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

Class(SMP_Unparser,      SMP_UnparseMixin, CUnparserProg);
Class(SMP_MacroUnparser, SMP_UnparseMixin, CMacroUnparserProg);

Class(OpenMP_Unparser,      OpenMP_UnparseMixin, CUnparserProg);
Class(OpenMP_MacroUnparser, OpenMP_UnparseMixin, CMacroUnparserProg);

# suggested values: bufIters=64 (16 for older machines), maxRank=1 (larger value increases search space)
# Example: opts := InitGTLibgen(64, 1)
#
InitGTLibgen := function(bufIters, maxRank, useComplex)
    local opts;
    LibgenHardcodeStrides();

    opts := CopyFields(InitLibgen(When(useComplex, CplxLibgenDefaults, LibgenDefaults)),
        rec(
            useDeref := true,
            breakdownRules := rec(
                GT  := [ CopyFields(GT_Base, rec(maxSize := 32)),
                         CopyFields(GT_BufReshape, rec(bufIters := bufIters)),
                         CopyFields(GT_DFT_CT, rec(minSize := 33, maxRank := maxRank)),
                         GT_NthLoop, GT_Par ],
                DFT := [ CopyFields(DFT_CT, rec(maxSize:=32)),
                         CopyFields(DFT_GT_CT, rec(minSize:=32)),
                         DFT_Base ],
                InfoNt := [Info_Base])
        ));
    opts.formulaStrategies.preRC := [ HfuncSumsRules ];
    return opts;
end;

InitSMPGTLibgen := function(bufIters, maxRank, useComplex, useOpenMP)
    local opts;
    opts := CopyFields(InitGTLibgen(bufIters, maxRank, useComplex), rec(
            unparser := Cond(
                useComplex and useOpenMP,         OpenMP_MacroUnparser,
                useComplex and not useOpenMP,        SMP_MacroUnparser,
                not useComplex and useOpenMP,     OpenMP_Unparser,
                not useComplex and not useOpenMP,    SMP_Unparser)));

    opts.formulaStrategies.sigmaSpl := [ MergedRuleSet(StandardSumsRules,RulesSMP), HfuncSumsRules ];
    opts.formulaStrategies.rc := opts.formulaStrategies.sigmaSpl;

    if not useOpenMP then
        opts.subParams := [var("num_threads", TInt), var("tid", TInt)];
        opts.profile := When(LocalConfig.osinfo.isWindows(),
            LocalConfig.cpuinfo.profile.threads(),
            profiler.default_profiles.linux_x86_threads
        );
    fi;
    return opts;
end;


# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

@TInt := @.cond(x->x.t=TInt);
@TReal := @.cond(x->IsRealT(x.t));
@_scalar := @.cond(x->IsOrdT(x.t) or IsRealT(x.t) or ObjId(x.t)=TPtr);
@TVect := @.cond(x->IsVecT(x.t));
@Value := @.cond(x->IsValue(x));
@nth := @.cond(x->ObjId(x)=nth);

_isa := self -> self.opts.vector.isa;
_epi := (self, o) -> Concat("epi", self.ctype_suffixval(o.t, _isa(self)));
_px := (self, o) -> self.ctype_suffixval(o.t, _isa(self));
_vp := (o) -> let(pp := Last(o.args),
    Cond(ObjId(pp)=vparam, pp.p,
	 IsValue(pp), pp.v,
	 pp));

Class(AVXUnparser, SSEUnparser, rec(
    # --------------------------------
    # ISA constructs, general
    # -------------------------------

    # This is a general suffix for intrinsics that is determine from the data type
    ctype_suffix := (self, t, isa) >> Cond(
        t = TVect(T_Real(64), 4), "pd",
        t = TVect(T_Real(32), 8), "ps",
        t = TVect(TReal, 4) and isa=AVX_4x64f, "pd",
        t = TVect(TReal, 8) and isa=AVX_8x32f, "ps",
        Inherited(t, isa)
    ),

    ctype_prefix := (self, t) >> Cond( _avxT(t, self.opts), "_mm256", "_mm" ),

    # This is the type used for declarations of vector variables
    ctype := (self, t, isa) >> Cond(
        t in [TReal, TVect(TReal, 1)],
          Cond(
            isa = AVX_4x64f, "double",
            isa = AVX_8x32f, "float",
            "UNKNOWN_TYPE"),
        t = T_Real(64), "double",
        t = T_Real(32), "float",
        # else
          Cond(
            t = TVect(TReal, 2),
              Cond(
                isa = AVX_4x64f, "__m128d",
                isa = AVX_8x32f, "__m64",
                "UNKNOWN_TYPE"),
            t = TVect(TReal, 4),
              Cond(
                isa = AVX_4x64f, "__m256d",
                isa = AVX_8x32f, "__m128",
                "UNKNOWN_TYPE"),
            t = TVect(TReal, 8),
              Cond(
                isa = AVX_8x32f, "__m256",
                "UNKNOWN_TYPE"),
            t = TVect(T_Real(64), 4), "__m256d",
            t = TVect(T_Real(32), 8), "__m256",
            t = TVect(T_Int(32),  8), "__m256i",
            t = TVect(T_UInt(32), 8), "__m256i",
            t = TVect(T_Real(64), 2), "__m128d",
            t = TVect(T_Real(32), 4), "__m128",
            t = TVect(T_Real(32), 2), "__m64",
            Inherited(t, isa))
    ),

    cvalue_suffix  := (self, t)  >> let( isa := _isa(self), Cond(
        (t = TReal and isa = AVX_8x32f) or t = T_Real(32), "f",
        (t = TReal and isa = AVX_4x64f) or t = T_Real(64), "",
        Inherited(t)
    )),

    vhex := (self, o, i, is) >> Print("_mm_set_", _epi(self, o), "(", self.infix(Reversed(o.p), ", "), ")"),

    vparam := (self, o, i, is) >> When(Length(o.p)=1, Print(o.p[1]), iclshuffle(o.p)),

    Value := (self, o, i, is) >> Cond(
        o.t = TString, Print(o.v),

        o.t = TReal or ObjId(o.t) = T_Real, let(v := When(IsCyc(o.v), ReComplex(Complex(o.v)), Double(o.v)),
            When(v<0, Print("(", v, self.cvalue_suffix(o.t), ")"), Print(v, self.cvalue_suffix(o.t)))),

        o.t = TComplex, Print("COMPLEX(", ReComplex(Complex(o.v)), self.cvalue_suffix(TReal), ", ", ImComplex(Complex(o.v)), self.cvalue_suffix(TReal), ")"),

	o.t in [TInt, TUChar, TChar],
            When(o.v < 0, Print("(", o.v, ")"), Print(o.v)),

        ObjId(o.t) = TVect and Length(Set(o.v)) = 1,
            Cond( self.cx.isInside(Value) and Length(self.cx.Value) >= 2, # nested in an array
		    Print("{", self.infix(Replicate(o.t.size, o.v[1]), ", "), "}"),
                  # else
                    Cond( _avxT(o.t, self.opts), let( sfx := self.ctype_suffix(o.t, _isa(self)), pfx := self.ctype_prefix(o.t),
                        Cond( o.v[1] = 0,
                            self.printf("$1_setzero_$2()", [pfx, sfx]),
                            self.printf("$1_set1_$2($3)", [pfx, sfx, o.v[1]]))),
                        Inherited(o, i, is))),
        ObjId(o.t) = TVect,
            Cond( self.cx.isInside(Value) and Length(self.cx.Value) >= 2, # nested in an array
		    Print("{", self.infix((o.v), ", "), "}"),
                # else
                    Cond( _avxT(o.t, self.opts), let( sfx := self.ctype_suffix(o.t, _isa(self)), pfx := self.ctype_prefix(o.t),
		        Print(pfx, "_set_", sfx, "(", self.infix(Reversed(o.v), ", "), ")")),
                        Inherited(o, i, is))),

        IsArray(o.t),
           Print("{", self.infix(o.v, ", "), "}"),

        ObjId(o.t) = TSym,
            Print("(", self.declare(o.t, [], 0, 0), ") ", o.v),

	o.t = TBool, Print(When(o.v, "1", "0")),

	#Error(self,".Value cannot unparse type ",o.t)
        Inherited(o, i, is)
    ),

    vpack := (self, o, i, is) >> Cond( _avxT(o.t, self.opts),
            Print("_mm256_set_", self.ctype_suffix(o.t, _isa(self)), "(", self.infix(Reversed(o.args), ", "), ")"),

        Inherited(o, i, is)),

    vdup := (self, o, i, is) >> CondPat(o,
            [vdup, @(1, [nth, deref]), @TInt], let( isa := _isa(self),
                t := o.args[1].t, pfx := self.ctype_prefix(o.t),
                Cond( t = T_Real(32) or (t=TReal and isa=AVX_8x32f),
                        self.printf("$1_broadcast_ss($2)", [pfx, o.args[1].toPtr(t)]),
                      t = T_Real(64) or (t=TReal and isa=AVX_4x64f),
                        self.printf("$1_broadcast_sd($2)", [pfx, o.args[1].toPtr(t)]),
                      t = TVect(T_Real(32), 2) or (t=TVect(TReal, 2) and isa=AVX_8x32f),
                        self.printf("$1_castpd_ps($1_broadcast_sd($2))", [pfx, o.args[1].toPtr(T_Real(64))]),
                      t = TVect(T_Real(64), 2) or (t=TVect(TReal, 2) and isa=AVX_4x64f),
                        self.printf("$1_broadcast_pd($2)", [pfx, o.args[1].toPtr(t)]),
                      t = TVect(T_Real(32), 4) or (t=TVect(TReal, 4) and isa=AVX_8x32f),
                        self.printf("$1_castpd_ps($1_broadcast_pd($2))", [pfx, o.args[1].toPtr(TVect(T_Real(64), 2))]),
                      Error("unexpected vdup load"))),
            #[vdup, @nth,@.cond(x->x.t=TInt and x.v=2)],
            #    Print("_mm_loaddup_", sfx, "(&(", self(o.args[1], i, is), "))"),
            [vdup, @, @TInt],
                Print(When(_isa(self).v = 4, "_mm256_set1_pd", "_mm256_set1_ps"), "(", self(o.args[1], i, is), ")")
        ),

    # Declarations
    TVect := (self, t, vars, i, is) >> let( ctype := self.ctype(t, _isa(self)),
        Print(ctype, " ", self.infix(vars, ", "))),

    TReal := ~.TVect,

    TInt := (self, t, vars, i, is) >> Print("int ", self.infix(vars, ", ")),

    TBool := (self, t, vars, i, is) >> Print("BOOL ", self.infix(vars, ", ")),

    # Arithmetic
    mul := (self, o, i, is) >> let(n := Length(o.args), Cond(
        not IsVecT(o.t),
            Print("(", self.pinfix(o.args, ")*("), ")"),
        not _avxT(o.t, self.opts),
            Inherited(o, i, is),
	n > 2 and n mod 2 <> 0,
            self(mul(o.args[1], ApplyFunc(mul, Drop(o.args, 1))), i, is),
        n > 2,
            self(mul(ApplyFunc(mul, o.args{[1..n/2]}), ApplyFunc(mul, o.args{[n/2+1..n]})), i, is),
        let( sfx := self.ctype_suffix(o.t, _isa(self)),
         CondPat(o,
           [mul, @TReal, @TVect],
              self(mul(vdup(o.args[1],o.t.size), o.args[2]), i, is),
           [mul, @TVect, @TReal],
              self(mul(o.args[1], vdup(o.args[2],o.t.size)), i, is),
           [mul, @TInt, @TVect],
              self(mul(vdup(_toReal(o.args[1]),o.t.size), o.args[2]), i, is),
           [mul, @TVect, @TInt],
              self(mul(o.args[1], vdup(_toReal(o.args[2]),o.t.size)), i, is),
           [mul, @TVect,   @TVect],
              self.printf("_mm256_mul_$1($2, $3)", [sfx, o.args[1], o.args[2]]),
           Error("Don't know how to unparse <o>. Unrecognized type combination")))
    )),

    # -- add --
    add := (self, o, i, is) >> let(n := Length(o.args), Cond(
	not IsVecT(o.t),
            self.pinfix(o.args, " + "),
        not _avxT(o.t, self.opts),
            Inherited(o, i, is),
        n > 2 and n mod 2 <> 0,
            self(add(o.args[1], ApplyFunc(add, Drop(o.args, 1))), i, is),
        n > 2,
            self(add(ApplyFunc(add, o.args{[1..n/2]}), ApplyFunc(add, o.args{[n/2+1..n]})), i, is),
        let(sfx := self.ctype_suffix(o.t, _isa(self)), saturated:= When(_isa(self).isFixedPoint and _isa(self).saturatedArithmetic, "s", ""),
          CondPat(o,
            [add, @TReal, @TVect],
                self(add(vdup(o.args[1],o.t.size), o.args[2]), i, is),
            [add, @TVect, @TReal],
                self(add(o.args[1], vdup(o.args[2],o.t.size)), i, is),
            [add, @TInt, @TVect],
                self(add(vdup(_toReal(o.args[1]),o.t.size), o.args[2]), i, is),
            [add, @TVect, @TInt],
                self(add(o.args[1], vdup(_toReal(o.args[2]),o.t.size)), i, is),
            [add, @TVect,   @TVect],
                self.printf("_mm256_add$1_$2($3, $4)", [saturated, sfx, o.args[1], o.args[2]]),
            Error("Don't know how to unparse <o>. Unrecognized type combination")))
    )),

    sub := (self, o, i, is) >> Cond( _avxT(o.t, self.opts), let(
            isa       := _isa(self),
            sfx       := self.ctype_suffix(o.t, isa),
            saturated := When(isa.isFixedPoint and isa.saturatedArithmetic, "s", ""),
            self.printf("_mm256_sub$1_$2($3, $4)", [saturated, sfx, o.args[1], o.args[2]])),
        # else
            Inherited(o, i, is)),

    neg := (self, o, i, is) >> Cond( _avxT(o.t, self.opts),
        self(mul(neg(o.t.one()), o.args[1]), i, is),
        Inherited(o, i, is)),

    stickyNeg := ~.neg,

    sqrt  := (self, o, i, is) >> Cond( _avxT(o.t, self.opts),
        Checked( IsRealT(o.t.t), self.printf("_mm256_sqrt_$1($2)", [self.ctype_suffix(o.t, _isa(self)), o.args[1]])),
        Inherited(o, i, is)),

    rsqrt := (self, o, i, is) >> Cond( _avxT(o.t, self.opts),
        let( sfx := self.ctype_suffix(o.t, _isa(self)),
            Checked( sfx="ps", self.printf("_mm256_rsqrt_ps($1)", [o.args[1]]))),
        Inherited(o, i, is)),

    max := (self, o, i, is) >> let(n := Length(o.args), Cond(
        not _avxT(o.t, self.opts),
            Inherited(o, i, is),
	n > 2 and n mod 2 <> 0,
            self(max(o.args[1], ApplyFunc(max, Drop(o.args, 1))), i, is),
        n > 2,
            self(max(ApplyFunc(max, o.args{[1..n/2]}), ApplyFunc(max, o.args{[n/2+1..n]})), i, is),
        let( sfx := self.ctype_suffix(o.t, _isa(self)),
         CondPat(o,
           [max, @TReal, @TVect],
              self(max(vdup(o.args[1],o.t.size), o.args[2]), i, is),
           [max, @TVect, @TReal],
              self(max(o.args[1], vdup(o.args[2],o.t.size)), i, is),
           [max, @TInt, @TVect],
              self(max(vdup(_toReal(o.args[1]),o.t.size), o.args[2]), i, is),
           [max, @TVect, @TInt],
              self(max(o.args[1], vdup(_toReal(o.args[2]),o.t.size)), i, is),
           [max, @TVect,   @TVect],
              self.printf("_mm256_max_$1($2, $3)", [sfx, o.args[1], o.args[2]]),
           Error("Don't know how to unparse <o>. Unrecognized type combination")))
    )),

    min := (self, o, i, is) >> let(n := Length(o.args), Cond(
        not _avxT(o.t, self.opts),
            Inherited(o, i, is),
	n > 2 and n mod 2 <> 0,
            self(min(o.args[1], ApplyFunc(min, Drop(o.args, 1))), i, is),
        n > 2,
            self(min(ApplyFunc(min, o.args{[1..n/2]}), ApplyFunc(min, o.args{[n/2+1..n]})), i, is),
        let( sfx := self.ctype_suffix(o.t, _isa(self)),
         CondPat(o,
           [min, @TReal, @TVect],
              self(min(vdup(o.args[1],o.t.size), o.args[2]), i, is),
           [min, @TVect, @TReal],
              self(min(o.args[1], vdup(o.args[2],o.t.size)), i, is),
           [min, @TInt, @TVect],
              self(min(vdup(_toReal(o.args[1]),o.t.size), o.args[2]), i, is),
           [min, @TVect, @TInt],
              self(min(o.args[1], vdup(_toReal(o.args[2]),o.t.size)), i, is),
           [min, @TVect,   @TVect],
              self.printf("_mm256_min_$1($2, $3)", [sfx, o.args[1], o.args[2]]),
           Error("Don't know how to unparse <o>. Unrecognized type combination")))
    )),

    # assuming we have ICC <ia32intrin.h> here
    log := (self, o, i, is) >> Cond( _avxT(o.t, self.opts), let( sfx := self.ctype_suffix(o.t, _isa(self)),
        Checked( sfx in ["ps", "pd"], Cond(
            Length(o.args)>1 and (o.args[2]=2 or o.args[2]=o.t.value(2)),
                self.printf("_mm256_log2_$1($2)", [sfx, o.args[1]]),
            Length(o.args)>1 and (o.args[2]=10 or o.args[2]=o.t.value(10)),
                self.printf("_mm256_log10_$1($2)", [sfx, o.args[1]]),
            Length(o.args)=1 or o.args[2]=d_exp(1) or o.args[2]=o.t.value(d_exp(1)),
                self.printf("_mm256_log_$1($2)", [sfx, o.args[1]]),
            self.printf("_mm256_div_$1(_mm256_log_$1($2), _mm256_log_$1($3))", [sfx, o.args[1]])))),
        Inherited(o, i, is)),

    # assuming we have ICC <ia32intrin.h> here
    exp := (self, o, i, is) >> Cond( _avxT(o.t, self.opts), let( sfx := self.ctype_suffix(o.t, _isa(self)),
        Checked( sfx in ["ps", "pd"], self.printf("_mm256_exp_$1($2)", [sfx, o.args[1]]))),
        Inherited(o, i, is)),

    # assuming we have ICC <ia32intrin.h> here
    pow := (self, o, i, is) >> Cond( _avxT(o.t, self.opts), let( sfx := self.ctype_suffix(o.t, _isa(self)),
        Checked( sfx in ["ps", "pd"], Cond(
            o.args[1]=2 or o.args[1]=o.t.value(2),
                self.printf("_mm256_exp2_$1($2)", [sfx, o.args[2]]),
            o.args[1]=d_exp(1) or o.args[1]=o.t.value(d_exp(1)),
                self.printf("_mm256_exp_$1($2)", [sfx, o.args[2]]),
            self.printf("_mm256_pow_$1($2, $3)", [sfx, o.args[1], o.args[2]])))),
        Inherited(o, i, is)),

    # --------------------------------
    # logic
    #
    bin_xor := (self, o, i, is) >> Cond( _avxT(o.t, self.opts),
        self.printf("_mm256_xor_$1($2, $3)", [self.ctype_suffix(o.t, _isa(self)), o.args[1], o.args[2]]),
        Inherited(o, i, is)),
    bin_and := (self, o, i, is) >> Cond( _avxT(o.t, self.opts),
        self.printf("_mm256_and_$1($2, $3)", [self.ctype_suffix(o.t, _isa(self)), o.args[1], o.args[2]]),
        Inherited(o, i, is)),
    bin_or := (self, o, i, is) >> Cond( _avxT(o.t, self.opts),
        self.printf("_mm256_or_$1($2, $3)", [self.ctype_suffix(o.t, _isa(self)), o.args[1], o.args[2]]),
        Inherited(o, i, is)),
    bin_andnot := (self, o, i, is) >> Cond( _avxT(o.t, self.opts),
        self.printf("_mm256_andnot_$1($2, $3)", [self.ctype_suffix(o.t, _isa(self)), o.args[1], o.args[2]]),
        Inherited(o, i, is)),

    # --------------------------------
    # ISA specific : AVX_4x64f
    #
    cmpge_4x64f := (self, o, i, is) >> self.prefix("_mm256_cmp_pd", Concat(o.args, ["_CMP_GE_OQ"])),
    logic_and_4x64f := (self, o, i, is) >> self.prefix("_mm256_and_pd", o.args),
    logic_xor_4x64f := (self, o, i, is) >> self.prefix("_mm256_xor_pd", o.args),

    vloadu_4x64f   := (self, o, i, is) >> self.prefix("_mm256_loadu_pd", o.args),
    vstoreu_4x64f  := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm256_storeu_pd", o.args), ";\n"),

    vinsert_2l_4x64f := (self, o, i, is) >> self.prefix("_mm256_insertf128_pd", o.args),
    vloadmask_4x64f  := (self, o, i, is) >> self.printf("_mm256_maskload_pd($1, _mm256_set_epi32($2))", [o.args[1], ()->PrintDel(_vp(o), ", ")]),
    vbroadcast_4x64f := (self, o, i, is) >> self.printf("_mm256_broadcast_sd($1)", [o.args[1]]),
    # doublecheck this
    vextract_2l_4x64f := (self, o, i, is) >> self.prefix("_mm256_extractf128_pd", o.args),

    vstore_2l_4x64f := (self, o, i, is) >> self.prefix("_mm256_extractf128_pd", o.args),
    vstoremask_4x64f := (self, o, i, is) >> Print(Blanks(i), self.printf("_mm256_maskstore_pd($1, _mm256_set_epi32($3), $2)", [o.args[1], o.args[2], ()->PrintDel(List(_vp(o), e->e.v), ", ")]), ";\n"),

    vunpacklo_4x64f := (self, o, i, is) >> self.prefix("_mm256_unpacklo_pd", o.args),
    vunpackhi_4x64f := (self, o, i, is) >> self.prefix("_mm256_unpackhi_pd", o.args),
    vpermf128_4x64f := (self, o, i, is) >> self.printf("_mm256_permute2f128_pd($1, $2, ($3) | (($4) << 4))",
        let(l := _vp(o)-1, [o.args[1], o.args[2], l[1], l[2]])),
    vshuffle_4x64f := (self, o, i, is) >> self.printf("_mm256_shuffle_pd($1, $2, ($3) | (($4) << 1) | (($5) << 2) | (($6) << 3))",
        let(l := _vp(o)-1, [o.args[1], o.args[2], l[1], l[2], l[3], l[4]])),
    vperm_4x64f := (self, o, i, is) >> self.printf("_mm256_permute_pd($1, ($2) | (($3) << 1) | (($4) << 2) | (($5) << 3))",
        let(l := _vp(o)-1, [o.args[1], l[1], l[2], l[3], l[4]])),
    vblend_4x64f := (self, o, i, is) >> self.printf("_mm256_blend_pd($1, $2, ($3) | (($4) << 1) | (($5) << 2) | (($6) << 3))",
        let(l := _vp(o)-1, [o.args[1], o.args[2], l[1], l[2], l[3], l[4]])),

    vuunpacklo_4x64f := (self, o, i, is) >> self(o.toBinop(), i, is),
    vuunpackhi_4x64f := (self, o, i, is) >> self(o.toBinop(), i, is),
    vupermf128_4x64f := (self, o, i, is) >> self(o.toBinop(), i, is),
    vushuffle_4x64f  := (self, o, i, is) >> self(o.toBinop(), i, is),
    vuperm2_4x64f    := (self, o, i, is) >> self(o.toBinop(), i, is),

    addsub_4x64f := (self, o, i, is) >> When(Length(o.args) > 2,
        Error("addsub_2x64f is strictly binary"),
        CondPat(o,
           [addsub_4x64f, @TReal, @TVect],
              self(_computeExpType(addsub_4x64f(vdup(o.args[1],o.t.size), o.args[2])), i, is),
           [addsub_4x64f, @TVect, @TReal],
              self(_computeExpType(addsub_4x64f(o.args[1], vdup(o.args[2],o.t.size))), i, is),
           [addsub_4x64f, @TInt, @TVect],
              self(_computeExpType(addsub_4x64f(vdup(_toReal(o.args[1]),o.t.size), o.args[2])), i, is),
           [addsub_4x64f, @TVect, @TInt],
              self(_computeExpType(addsub_4x64f(o.args[1], vdup(_toReal(o.args[2]),o.t.size))), i, is),
           [addsub_4x64f, @TVect,   @TVect],
              self.printf("_mm256_addsub_pd($1, $2)", [o.args[1], o.args[2]]),
           Error("Don't know how to unparse <o>. Unrecognized type combination")
        )),

    fmaddsub_4x64f := (self, o, i, is) >> When(
        Length(o.args) <> 3, Error("fmaddsub_4x64f is strictly ternary"),
        CondPat(o,
           [fmaddsub_4x64f, @TVect, @TVect, @TVect],
              self.printf("_mm256_fmaddsub_pd($1, $2, $3, 0)", [o.args[1], o.args[2], o.args[3]]),
           Error("Don't know how to unparse <o>. Unrecognized type combination")
        )),

    vzero_4x64f := (self, o, i, is) >> Print("_mm256_setzero_pd()"),

    # --------------------------------
    # ISA specific : AVX_8x32f
    #
    logic_and_8x32f := (self, o, i, is) >> self.prefix("_mm256_and_ps", o.args),
    logic_xor_8x32f := (self, o, i, is) >> self.prefix("_mm256_xor_ps", o.args),

    vloadu_8x32f      := (self, o, i, is) >> self.prefix("_mm256_loadu_ps", o.args),
    vstoreu_8x32f     := (self, o, i, is) >> Print(Blanks(i), self.prefix("_mm256_storeu_ps", o.args), ";\n"),

    vinsert_4l_8x32f  := (self, o, i, is) >> self.prefix("_mm256_insertf128_ps", o.args),
    vloadmask_8x32f   := (self, o, i, is) >> self.printf("_mm256_maskload_ps($1, _mm256_set_epi32($2))", [o.args[1], ()->PrintDel(_vp(o), ", ")]),
    vhdup_8x32f       := (self, o, i, is) >> self.printf("_mm256_movehdup_ps($1)", [o.args[1]]),
    vldup_8x32f       := (self, o, i, is) >> self.printf("_mm256_moveldup_ps($1)", [o.args[1]]),

    vextract_4l_8x32f := (self, o, i, is) >> self.prefix("_mm256_extractf128_ps", o.args),
    vstore_4l_8x32f   := (self, o, i, is) >> self.prefix("_mm256_extractf128_ps", o.args),
    vstoremask_8x32f  := (self, o, i, is) >> Print(Blanks(i), self.printf("_mm256_maskstore_ps($1, _mm256_set_epi32($3), $2)", [o.args[1], o.args[2], ()->PrintDel(List(_vp(o), e->e.v), ", ")]), ";\n"),

    vunpacklo_8x32f   := (self, o, i, is) >> self.prefix("_mm256_unpacklo_ps", o.args),
    vunpackhi_8x32f   := (self, o, i, is) >> self.prefix("_mm256_unpackhi_ps", o.args),

    vpermf128_8x32f   := (self, o, i, is) >> self.printf("_mm256_permute2f128_ps($1, $2, ($3) | (($4) << 4))",
        let(l := _vp(o)-1, [o.args[1], o.args[2], l[1], l[2]])),

    vshuffle_8x32f    := (self, o, i, is) >> self.printf("_mm256_shuffle_ps($1, $2, ($3) | (($4) << 2) | (($5) << 4) | (($6) << 6))",
        let(l := _vp(o)-1, [o.args[1], o.args[2], l[1], l[2], l[3], l[4]])),

    vperm_8x32f       := (self, o, i, is) >> self.printf("_mm256_permute_ps($1, ($2) | (($3) << 2) | (($4) << 4) | (($5) << 6))",
        let(l := _vp(o)-1, [o.args[1], l[1], l[2], l[3], l[4]])),
    vblend_8x32f := (self, o, i, is) >> self.printf("_mm256_blend_ps($1, $2, ($3) | (($4) << 1) | (($5) << 2) | (($6) << 3) | (($7) << 4) | (($8) << 5) | (($9) << 6) | (($10) << 7))",
        let(l := _vp(o)-1, [o.args[1], o.args[2], l[1], l[2], l[3], l[4], l[5], l[6], l[7], l[8]])),

    vuunpacklo_8x32f := (self, o, i, is) >> self(o.toBinop(), i, is),
    vuunpackhi_8x32f := (self, o, i, is) >> self(o.toBinop(), i, is),
    vupermf128_8x32f := (self, o, i, is) >> self(o.toBinop(), i, is),
    vushuffle_8x32f  := (self, o, i, is) >> self(o.toBinop(), i, is),

    addsub_8x32f := (self, o, i, is) >> When(Length(o.args) > 2,
        Error("addsub_2x64f is strictly binary"),
        CondPat(o,
           [addsub_8x32f, @TReal, @TVect],
              self(_computeExpType(addsub_8x32f(vdup(o.args[1],o.t.size), o.args[2])), i, is),
           [addsub_8x32f, @TVect, @TReal],
              self(_computeExpType(addsub_8x32f(o.args[1], vdup(o.args[2],o.t.size))), i, is),
           [addsub_8x32f, @TInt, @TVect],
              self(_computeExpType(addsub_8x32f(vdup(_toReal(o.args[1]),o.t.size), o.args[2])), i, is),
           [addsub_8x32f, @TVect, @TInt],
              self(_computeExpType(addsub_8x32f(o.args[1], vdup(_toReal(o.args[2]),o.t.size))), i, is),
           [addsub_8x32f, @TVect,   @TVect],
              self.printf("_mm256_addsub_ps($1, $2)", [o.args[1], o.args[2]]),
           Error("Don't know how to unparse <o>. Unrecognized type combination")
        )),

    fmaddsub_8x32f := (self, o, i, is) >> When(
        Length(o.args) <> 3, Error("fmaddsub_8x32f is strictly ternary"),
        CondPat(o,
           [fmaddsub_8x32f, @TVect, @TVect, @TVect],
              self.printf("_mm256_fmaddsub_ps($1, $2, $3, 0)", [o.args[1], o.args[2], o.args[3]]),
           Error("Don't know how to unparse <o>. Unrecognized type combination")
        )),

    vzero_8x32f := (self, o, i, is) >> Print("_mm256_setzero_ps()"),

    #--------------------------------
    # Conversion

    vcvt_8x32f_4x64f  := (self, o, i, is) >> self.printf("_mm256_cvtpd_ps($1)", [o.args[1]]),
    vcvt_4x64f_4x32f  := (self, o, i, is) >> self.printf("_mm256_cvtps_pd($1)", [o.args[1]]),
    vcvt_4x64f_4x32i  := (self, o, i, is) >> self.printf("_mm256_cvtepi32_pd($1)", [o.args[1]]),
    vcvt_4x32i_4x64f  := (self, o, i, is) >> self.printf("_mm256_cvtpd_epi32($1)", [o.args[1]]),
    vcvtt_4x32i_4x64f := (self, o, i, is) >> self.printf("_mm256_cvttpd_epi32($1)", [o.args[1]]),
    vcvt_8x32f_8x32i  := (self, o, i, is) >> self.printf("_mm256_cvtepi32_ps($1)", [o.args[1]]),
    vcvt_8x32i_8x32f  := (self, o, i, is) >> self.printf("_mm256_cvtps_epi32($1)", [o.args[1]]),
    vcvtt_8x32i_8x32f := (self, o, i, is) >> self.printf("_mm256_cvttps_epi32($1)", [o.args[1]]),

));


# Copyright (c) 2018-2021, Carnegie Mellon University
# See LICENSE for details

#   FF: needed to merge integer tables due to SAR grds. not sure where to put yet...
MergeIntData := function(s, opts)
    local datas, first, eqdata, subst;
    datas := Filtered(Set(Collect(s, @(1, var, e->IsBound(e.value) and not IsLoopIndex(e)))), i->i.t.t=TInt);

    while datas <> [] do
        first := datas[1];
        datas := Drop(datas, 1);
        eqdata := Filtered(datas, i->i.value=first.value);
        if Length(eqdata) > 0 then
            SubtractSet(datas, Set(eqdata));
            for subst in eqdata do
                SubstVars(s, rec((subst.id) := first));
            od;
        fi;
    od;
    return s;
end;