| /* |
| * Copyright 2023 Google LLC |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package kelvin |
| |
| import chisel3._ |
| import chisel3.util._ |
| import _root_.circt.stage.ChiselStage |
| |
| // VAluInt is foremost an ML depthwise and activiation unit with pipelining |
| // behaviors optimized to this functionality. All operations are pipelined with |
| // a result latency of 2cc geared towards the goal of simplicity of design. |
| // |
| // Note: widening operations modify the size from ISA defined destination to |
| // source read registers of sz/2. |
| |
| class VAluInt(p: Parameters, aluid: Int) extends Module { |
| val e = new VEncodeOp() |
| |
| val io = IO(new Bundle { |
| val in = Input(new Bundle { |
| val valid = Bool() |
| val op = UInt(e.bits.W) |
| val f2 = UInt(3.W) |
| val sz = UInt(3.W) |
| val vd = new AluAddr() // write port 0 |
| val ve = new AluAddr() // write port 1 |
| val sv = new Bundle { val data = UInt(32.W) } // scala value |
| }) |
| val read = Vec(7, Input(new Bundle { |
| val data = UInt(p.vectorBits.W) |
| })) |
| val write = Vec(2, Output(new Bundle { |
| val valid = Bool() |
| val addr = UInt(6.W) |
| val data = UInt(p.vectorBits.W) |
| })) |
| val whint = Vec(2, Output(new Bundle { |
| val valid = Bool() |
| val addr = UInt(6.W) |
| })) |
| }) |
| |
| class AluAddr extends Bundle { |
| val addr = UInt(6.W) |
| } |
| |
| val lanes = p.vectorBits / 32 |
| assert(lanes == 4 || lanes == 8 || lanes == 16) |
| |
| assert(!io.in.valid || PopCount(io.in.sz) <= 1.U) |
| |
| // --------------------------------------------------------------------------- |
| // Tie-offs. |
| for (i <- 0 until 2) { |
| io.write(i).valid := false.B |
| io.write(i).addr := 0.U |
| io.write(i).data := 0.U |
| } |
| for (i <- 0 until 2) { |
| io.whint(i).valid := false.B |
| io.whint(i).addr := 0.U |
| } |
| |
| // --------------------------------------------------------------------------- |
| // Encodings. |
| val e_absd = io.in.op === e.vabsd.U |
| val e_acc = io.in.op === e.vacc.U |
| val e_dup = io.in.op === e.vdup.U |
| val e_max = io.in.op === e.vmax.U |
| val e_min = io.in.op === e.vmin.U |
| val e_rsub = io.in.op === e.vrsub.U |
| val e_srans = io.in.op === e.vsrans.U |
| val e_sraqs = if (aluid == 0) io.in.op === e.vsraqs.U else false.B |
| |
| val e_slidevn = io.in.op === e.vslidevn.U || io.in.op === e.vslidehn.U || io.in.op === e.vslidehn2.U |
| val e_slidevp = io.in.op === e.vslidevp.U || io.in.op === e.vslidehp.U || io.in.op === e.vslidehp2.U |
| val e_slidehn2 = io.in.op === e.vslidehn2.U |
| val e_slidehp2 = io.in.op === e.vslidehp2.U |
| val e_sel = io.in.op === e.vsel.U |
| val e_evn = io.in.op === e.vevn.U || io.in.op === e.vevnodd.U |
| val e_odd = io.in.op === e.vodd.U || io.in.op === e.vevnodd.U |
| val e_zip = io.in.op === e.vzip.U |
| |
| val e_dwinit = io.in.op === e.adwinit.U |
| val e_dwconv = io.in.op === e.vdwconv.U || io.in.op === e.adwconv.U |
| val e_dwconva = io.in.op === e.adwconv.U |
| |
| val e_add_add = io.in.op === e.vadd.U |
| val e_add_adds = io.in.op === e.vadds.U |
| val e_add_addw = io.in.op === e.vaddw.U |
| val e_add_add3 = io.in.op === e.vadd3.U |
| val e_add_hadd = io.in.op === e.vhadd.U |
| val e_add = e_add_add || e_add_adds || e_add_addw || e_add_add3 || e_add_hadd |
| |
| val e_cmp_eq = io.in.op === e.veq.U |
| val e_cmp_ne = io.in.op === e.vne.U |
| val e_cmp_lt = io.in.op === e.vlt.U |
| val e_cmp_le = io.in.op === e.vle.U |
| val e_cmp_gt = io.in.op === e.vgt.U |
| val e_cmp_ge = io.in.op === e.vge.U |
| val e_cmp = e_cmp_eq || e_cmp_ne || e_cmp_lt || e_cmp_le || e_cmp_gt || e_cmp_ge |
| assert(PopCount(Cat(e_cmp_eq, e_cmp_ne, e_cmp_lt, e_cmp_le, e_cmp_gt, e_cmp_ge)) <= 1.U) |
| |
| val e_log_and = io.in.op === e.vand.U |
| val e_log_or = io.in.op === e.vor.U |
| val e_log_xor = io.in.op === e.vxor.U |
| val e_log_not = io.in.op === e.vnot.U |
| val e_log_rev = io.in.op === e.vrev.U |
| val e_log_ror = io.in.op === e.vror.U |
| val e_log_clb = io.in.op === e.vclb.U |
| val e_log_clz = io.in.op === e.vclz.U |
| val e_log_cpop = io.in.op === e.vcpop.U |
| val e_log = e_log_and || e_log_or || e_log_xor || e_log_not || e_log_rev || e_log_ror || e_log_clb || e_log_clz || e_log_cpop |
| assert(PopCount(Cat(e_log_and, e_log_or, e_log_xor, e_log_not, e_log_rev, e_log_ror, e_log_clb, e_log_clz, e_log_cpop)) <= 1.U) |
| |
| val e_mul0_dmulh = io.in.op === e.vdmulh.U || io.in.op === e.vdmulh2.U |
| val e_mul0_mul = io.in.op === e.vmul.U || io.in.op === e.vmul2.U |
| val e_mul0_mulh = io.in.op === e.vmulh.U || io.in.op === e.vmulh2.U |
| val e_mul0_muls = io.in.op === e.vmuls.U || io.in.op === e.vmuls2.U |
| val e_mul0_mulw = io.in.op === e.vmulw.U |
| val e_mul0_madd = io.in.op === e.vmadd.U |
| val e_mul0 = e_mul0_dmulh || e_mul0_mul || e_mul0_mulh || e_mul0_muls || e_mul0_mulw || e_mul0_madd |
| |
| val e_mul1_dmulh = io.in.op === e.vdmulh2.U |
| val e_mul1_mul = io.in.op === e.vmul2.U |
| val e_mul1_mulh = io.in.op === e.vmulh2.U |
| val e_mul1_muls = io.in.op === e.vmuls2.U |
| val e_mul1 = e_mul1_dmulh || e_mul1_mul || e_mul1_mulh || e_mul1_muls |
| |
| val e_mv2 = io.in.op === e.vmv2.U |
| val e_mvp = io.in.op === e.vmvp.U |
| val e_mv = io.in.op === e.vmv.U || e_mv2 || e_mvp |
| |
| val e_padd_add = io.in.op === e.vpadd.U |
| val e_padd_sub = io.in.op === e.vpsub.U |
| val e_padd = e_padd_add || e_padd_sub |
| |
| val e_shf_shl = io.in.op === e.vshl.U |
| val e_shf_shr = io.in.op === e.vshr.U |
| val e_shf_shf = io.in.op === e.vshf.U |
| val e_shf_l = e_shf_shl || e_shf_shf |
| val e_shf_r = e_shf_shr || e_shf_shf |
| |
| val e_sub_sub = io.in.op === e.vsub.U |
| val e_sub_subs = io.in.op === e.vsubs.U |
| val e_sub_subw = io.in.op === e.vsubw.U |
| val e_sub_hsub = io.in.op === e.vhsub.U |
| val e_sub = e_sub_sub || e_sub_subs || e_sub_subw || e_sub_hsub |
| |
| val e_negative = io.in.f2(0) && e_mul0_dmulh |
| val e_round = io.in.f2(1) && (e_add_hadd || e_sub_hsub || e_mul0_dmulh || e_mul0_mulh || e_shf_shf || e_srans || e_sraqs) |
| val e_signed = !io.in.f2(0) || e_mul0_dmulh |
| |
| assert(!(e_mul1_dmulh && !e_mul0_dmulh)) |
| assert(!(e_mul1_mul && !e_mul0_mul)) |
| assert(!(e_mul1_mulh && !e_mul0_mulh)) |
| assert(!(e_mul1_muls && !e_mul0_muls)) |
| |
| // --------------------------------------------------------------------------- |
| // Control. |
| val vdvalid0 = RegInit(false.B) |
| val vdvalid1 = RegInit(false.B) |
| val vevalid0 = RegInit(false.B) |
| val vevalid1 = RegInit(false.B) |
| val wmask = RegInit(false.B) |
| val vdaddr0 = Reg(new AluAddr()) |
| val vdaddr1 = Reg(new AluAddr()) |
| val veaddr0 = Reg(new AluAddr()) |
| val veaddr1 = Reg(new AluAddr()) |
| val sz = RegInit(0.U(3.W)) |
| val f2 = RegInit(0.U(3.W)) |
| val sv = RegInit(0.U(32.W)) |
| |
| when (io.in.valid) { |
| // Note: sz is source size, not destination as is ISA defined. |
| val nxt_vdvalid = e_dwconv || e_mul0 || e_absd || e_acc || e_add || e_cmp || e_dup || e_log || e_evn || e_max || e_min || e_mv || e_padd || e_rsub || e_sel || e_shf_l || e_shf_r || e_slidevn || e_slidevp || e_srans || e_sraqs || e_sub || e_zip |
| val nxt_vevalid = e_dwconv || e_mul1 || e_mul0_mulw || e_acc || e_add_addw || e_mv2 || e_mvp || e_odd || e_slidehn2 || e_slidehp2 || e_sub_subw || e_zip |
| val nxt_widen = e_acc || e_add_addw || e_mul0_mulw || e_sub_subw |
| vdvalid0 := nxt_vdvalid |
| vevalid0 := nxt_vevalid |
| wmask := e_dwconva |
| sz := MuxOR(nxt_vdvalid || nxt_vevalid, Mux(nxt_widen, io.in.sz >> 1.U, io.in.sz)) |
| f2 := io.in.f2 |
| sv := io.in.sv.data |
| } .elsewhen (vdvalid0 || vevalid0) { |
| vdvalid0 := false.B |
| vevalid0 := false.B |
| wmask := false.B |
| sz := 0.U |
| f2 := 0.U |
| sv := 0.U |
| } |
| |
| // Register VAluIntLane results, but mask io.write.valid outputs. |
| vdvalid1 := vdvalid0 && !wmask |
| vevalid1 := vevalid0 && !wmask |
| |
| when (io.in.valid) { |
| vdaddr0 := io.in.vd |
| veaddr0 := io.in.ve |
| } |
| |
| when (vdvalid0) { |
| vdaddr1 := vdaddr0 |
| } |
| |
| when (vevalid0) { |
| veaddr1 := veaddr0 |
| } |
| |
| // --------------------------------------------------------------------------- |
| // Side-bands. |
| val negative = Reg(Bool()) |
| val round = Reg(Bool()) |
| val signed = Reg(Bool()) |
| |
| when (io.in.valid) { |
| negative := e_negative |
| round := e_round |
| signed := e_signed |
| } |
| |
| // --------------------------------------------------------------------------- |
| // Operations. |
| val absd = Reg(Bool()) |
| val acc = Reg(Bool()) |
| val dup = Reg(Bool()) |
| val max = Reg(Bool()) |
| val min = Reg(Bool()) |
| val srans = Reg(Bool()) |
| val sraqs = Reg(Bool()) |
| |
| val slidevn = Reg(Bool()) |
| val slidevp = Reg(Bool()) |
| val slidehn2 = Reg(Bool()) |
| val slidehp2 = Reg(Bool()) |
| val sel = Reg(Bool()) |
| val evn = Reg(Bool()) |
| val odd = Reg(Bool()) |
| val zip = Reg(Bool()) |
| |
| val dwinit = Reg(Bool()) |
| val dwconv = Reg(Bool()) |
| val dwconvData = Reg(Bool()) |
| |
| val add = Reg(Bool()) |
| val add_add = Reg(Bool()) |
| val add_adds = Reg(Bool()) |
| val add_addw = Reg(Bool()) |
| val add_add3 = Reg(Bool()) |
| val add_hadd = Reg(Bool()) |
| |
| val padd = Reg(Bool()) |
| val padd_add = Reg(Bool()) |
| val padd_sub = Reg(Bool()) |
| |
| val rsub = Reg(Bool()) |
| val rsub_rsub = Reg(Bool()) |
| |
| val sub = Reg(Bool()) |
| val sub_sub = Reg(Bool()) |
| val sub_subs = Reg(Bool()) |
| val sub_subw = Reg(Bool()) |
| val sub_hsub = Reg(Bool()) |
| |
| val cmp = Reg(Bool()) |
| val cmp_eq = Reg(Bool()) |
| val cmp_ne = Reg(Bool()) |
| val cmp_lt = Reg(Bool()) |
| val cmp_le = Reg(Bool()) |
| val cmp_gt = Reg(Bool()) |
| val cmp_ge = Reg(Bool()) |
| |
| val log = Reg(Bool()) |
| val log_and = Reg(Bool()) |
| val log_or = Reg(Bool()) |
| val log_xor = Reg(Bool()) |
| val log_not = Reg(Bool()) |
| val log_rev = Reg(Bool()) |
| val log_ror = Reg(Bool()) |
| val log_clb = Reg(Bool()) |
| val log_clz = Reg(Bool()) |
| val log_cpop = Reg(Bool()) |
| |
| val mul0 = Reg(Bool()) |
| val mul0_dmulh = Reg(Bool()) |
| val mul0_mul = Reg(Bool()) |
| val mul0_mulh = Reg(Bool()) |
| val mul0_muls = Reg(Bool()) |
| val mul0_mulw = Reg(Bool()) |
| val mul0_madd = Reg(Bool()) |
| |
| val mul1 = Reg(Bool()) |
| val mul1_dmulh = Reg(Bool()) |
| val mul1_mul = Reg(Bool()) |
| val mul1_mulh = Reg(Bool()) |
| val mul1_muls = Reg(Bool()) |
| |
| val mv = Reg(Bool()) |
| val mv2 = Reg(Bool()) |
| val mvp = Reg(Bool()) |
| |
| val shf_l = Reg(Bool()) |
| val shf_r = Reg(Bool()) |
| val shf_shl = Reg(Bool()) |
| val shf_shr = Reg(Bool()) |
| val shf_shf = Reg(Bool()) |
| |
| val validClr = RegInit(false.B) |
| validClr := io.in.valid |
| |
| when (io.in.valid || validClr) { |
| val valid = io.in.valid |
| |
| absd := valid && e_absd |
| acc := valid && e_acc |
| dup := valid && e_dup |
| max := valid && e_max |
| min := valid && e_min |
| srans := valid && e_srans |
| sraqs := valid && e_sraqs |
| |
| slidevn := valid && e_slidevn |
| slidevp := valid && e_slidevp |
| slidehn2 := valid && e_slidehn2 |
| slidehp2 := valid && e_slidehp2 |
| sel := valid && e_sel |
| evn := valid && e_evn |
| odd := valid && e_odd |
| zip := valid && e_zip |
| |
| dwinit := valid && e_dwinit |
| dwconv := valid && e_dwconv |
| |
| add := valid && e_add // unit activation |
| add_add := valid && e_add_add |
| add_adds := valid && e_add_adds |
| add_addw := valid && e_add_addw |
| add_add3 := valid && e_add_add3 |
| add_hadd := valid && e_add_hadd |
| |
| padd := valid && e_padd |
| padd_add := valid && e_padd_add |
| padd_sub := valid && e_padd_sub |
| |
| cmp := valid && (e_cmp || e_absd || e_max || e_min) // unit activation |
| cmp_eq := valid && e_cmp_eq |
| cmp_ne := valid && e_cmp_ne |
| cmp_lt := valid && e_cmp_lt |
| cmp_le := valid && e_cmp_le |
| cmp_gt := valid && e_cmp_gt |
| cmp_ge := valid && e_cmp_ge |
| |
| log := valid && e_log // unit activation |
| log_and := valid && e_log_and |
| log_or := valid && e_log_or |
| log_xor := valid && e_log_xor |
| log_not := valid && e_log_not |
| log_rev := valid && e_log_rev |
| log_ror := valid && e_log_ror |
| log_clb := valid && e_log_clb |
| log_clz := valid && e_log_clz |
| log_cpop := valid && e_log_cpop |
| |
| mul0 := valid && e_mul0 // unit activation |
| mul0_dmulh := valid && e_mul0_dmulh |
| mul0_mul := valid && e_mul0_mul |
| mul0_mulh := valid && e_mul0_mulh |
| mul0_muls := valid && e_mul0_muls |
| mul0_mulw := valid && e_mul0_mulw |
| mul0_madd := valid && e_mul0_madd |
| |
| mul1 := valid && e_mul1 // unit activation |
| mul1_dmulh := valid && e_mul1_dmulh |
| mul1_mul := valid && e_mul1_mul |
| mul1_mulh := valid && e_mul1_mulh |
| mul1_muls := valid && e_mul1_muls |
| |
| mv := valid && e_mv |
| mv2 := valid && e_mv2 |
| mvp := valid && e_mvp |
| |
| rsub := valid && (e_rsub || e_absd) // unit activation |
| rsub_rsub := valid && e_rsub |
| |
| shf_l := valid && e_shf_l // unit activation |
| shf_r := valid && e_shf_r // unit activation |
| shf_shl := valid && e_shf_shl |
| shf_shr := valid && e_shf_shr |
| shf_shf := valid && e_shf_shf |
| |
| sub := valid && (e_sub || e_absd) |
| sub_sub := valid && e_sub_sub |
| sub_subs := valid && e_sub_subs |
| sub_subw := valid && e_sub_subw |
| sub_hsub := valid && e_sub_hsub |
| } |
| |
| // Second cycle of ALU pipeline. |
| dwconvData := dwconv |
| |
| // --------------------------------------------------------------------------- |
| // ALU segments. |
| val valu = for (i <- 0 until lanes) yield { |
| Module(new VAluIntLane) |
| } |
| |
| val load = Wire(Vec(2, UInt(p.vectorBits.W))) |
| |
| for (i <- 0 until lanes) { |
| val msb = 32 * i + 31 |
| val lsb = 32 * i |
| valu(i).io.in.vdvalid := vdvalid0 |
| valu(i).io.in.vevalid := vevalid0 |
| valu(i).io.in.sz := sz |
| for (j <- 0 until 7) { |
| valu(i).io.read(j).data := io.read(j).data(msb, lsb) |
| } |
| for (j <- 0 until 2) { |
| valu(i).io.load(j) := load(j)(msb, lsb) |
| } |
| } |
| |
| for (i <- 0 until lanes) { |
| valu(i).io.in.negative := negative |
| valu(i).io.in.round := round |
| valu(i).io.in.signed := signed |
| } |
| |
| for (i <- 0 until lanes) { |
| valu(i).io.op.absd := absd |
| valu(i).io.op.acc := acc |
| valu(i).io.op.dup := dup |
| valu(i).io.op.max := max |
| valu(i).io.op.min := min |
| valu(i).io.op.mv := mv |
| valu(i).io.op.mv2 := mv2 |
| valu(i).io.op.mvp := mvp |
| valu(i).io.op.srans := srans |
| valu(i).io.op.sraqs := sraqs |
| |
| valu(i).io.op.dwinit := dwinit |
| valu(i).io.op.dwconv := dwconv |
| valu(i).io.op.dwconvData := dwconvData |
| |
| valu(i).io.op.add.en := add |
| valu(i).io.op.add.add := add_add |
| valu(i).io.op.add.adds := add_adds |
| valu(i).io.op.add.addw := add_addw |
| valu(i).io.op.add.add3 := add_add3 |
| valu(i).io.op.add.hadd := add_hadd |
| |
| valu(i).io.op.cmp.en := cmp |
| valu(i).io.op.cmp.eq := cmp_eq |
| valu(i).io.op.cmp.ne := cmp_ne |
| valu(i).io.op.cmp.lt := cmp_lt |
| valu(i).io.op.cmp.le := cmp_le |
| valu(i).io.op.cmp.gt := cmp_gt |
| valu(i).io.op.cmp.ge := cmp_ge |
| |
| valu(i).io.op.log.en := log |
| valu(i).io.op.log.and := log_and |
| valu(i).io.op.log.or := log_or |
| valu(i).io.op.log.xor := log_xor |
| valu(i).io.op.log.not := log_not |
| valu(i).io.op.log.rev := log_rev |
| valu(i).io.op.log.ror := log_ror |
| valu(i).io.op.log.clb := log_clb |
| valu(i).io.op.log.clz := log_clz |
| valu(i).io.op.log.cpop := log_cpop |
| |
| valu(i).io.op.mul0.en := mul0 |
| valu(i).io.op.mul0.dmulh := mul0_dmulh |
| valu(i).io.op.mul0.mul := mul0_mul |
| valu(i).io.op.mul0.mulh := mul0_mulh |
| valu(i).io.op.mul0.muls := mul0_muls |
| valu(i).io.op.mul0.mulw := mul0_mulw |
| valu(i).io.op.mul0.madd := mul0_madd |
| |
| valu(i).io.op.mul1.en := mul1 |
| valu(i).io.op.mul1.dmulh := mul1_dmulh |
| valu(i).io.op.mul1.mul := mul1_mul |
| valu(i).io.op.mul1.mulh := mul1_mulh |
| valu(i).io.op.mul1.muls := mul1_muls |
| |
| valu(i).io.op.padd.en := padd |
| valu(i).io.op.padd.add := padd_add |
| valu(i).io.op.padd.sub := padd_sub |
| |
| valu(i).io.op.rsub.en := rsub |
| valu(i).io.op.rsub.rsub := rsub_rsub |
| |
| valu(i).io.op.shf.en.l := shf_l |
| valu(i).io.op.shf.en.r := shf_r |
| valu(i).io.op.shf.shl := shf_shl |
| valu(i).io.op.shf.shr := shf_shr |
| valu(i).io.op.shf.shf := shf_shf |
| |
| valu(i).io.op.sub.en := sub |
| valu(i).io.op.sub.sub := sub_sub |
| valu(i).io.op.sub.subs := sub_subs |
| valu(i).io.op.sub.subw := sub_subw |
| valu(i).io.op.sub.hsub := sub_hsub |
| } |
| |
| // --------------------------------------------------------------------------- |
| // VSlide. |
| def VSliden(sz: Int, sel: UInt, a: UInt, b: UInt): UInt = { |
| val size = 8 << sz |
| assert(sz == 0 || sz == 1 || sz == 2) |
| assert(size == 8 || size == 16 || size == 32) |
| assert(sel.getWidth == 2) |
| |
| val cnt = a.getWidth / size |
| val cnt2 = cnt * 2 |
| val in = Wire(Vec(cnt2, UInt(size.W))) |
| val sout1 = Wire(Vec(cnt, UInt(size.W))) |
| val sout2 = Wire(Vec(cnt, UInt(size.W))) |
| val sout3 = Wire(Vec(cnt, UInt(size.W))) |
| val sout4 = Wire(Vec(cnt, UInt(size.W))) |
| |
| for (i <- 0 until cnt) { |
| val l = i * size // lsb |
| val m = l + size - 1 // msb |
| in(i) := a(m,l) |
| in(i + cnt) := b(m,l) |
| } |
| |
| for (i <- 0 until cnt) { |
| sout1(i) := in(i + 1) |
| sout2(i) := in(i + 2) |
| sout3(i) := in(i + 3) |
| sout4(i) := in(i + 4) |
| } |
| |
| val out = MuxOR(sel === 0.U, sout1.asUInt) | |
| MuxOR(sel === 1.U, sout2.asUInt) | |
| MuxOR(sel === 2.U, sout3.asUInt) | |
| MuxOR(sel === 3.U, sout4.asUInt) |
| assert(out.getWidth == a.getWidth) |
| |
| out |
| } |
| |
| def VSlidep(sz: Int, sel: UInt, a: UInt, b: UInt): UInt = { |
| val size = 8 << sz |
| assert(sz == 0 || sz == 1 || sz == 2) |
| assert(size == 8 || size == 16 || size == 32) |
| assert(sel.getWidth == 2) |
| |
| val cnt = a.getWidth / size |
| val cnt2 = cnt * 2 |
| val in = Wire(Vec(cnt2, UInt(size.W))) |
| val sout1 = Wire(Vec(cnt, UInt(size.W))) |
| val sout2 = Wire(Vec(cnt, UInt(size.W))) |
| val sout3 = Wire(Vec(cnt, UInt(size.W))) |
| val sout4 = Wire(Vec(cnt, UInt(size.W))) |
| |
| for (i <- 0 until cnt) { |
| val l = i * size // lsb |
| val m = l + size - 1 // msb |
| in(i) := a(m,l) |
| in(i + cnt) := b(m,l) |
| } |
| |
| for (i <- 0 until cnt) { |
| sout1(i) := in(i - 1 + cnt) |
| sout2(i) := in(i - 2 + cnt) |
| sout3(i) := in(i - 3 + cnt) |
| sout4(i) := in(i - 4 + cnt) |
| } |
| |
| val out = MuxOR(sel === 0.U, sout1.asUInt) | |
| MuxOR(sel === 1.U, sout2.asUInt) | |
| MuxOR(sel === 2.U, sout3.asUInt) | |
| MuxOR(sel === 3.U, sout4.asUInt) |
| assert(out.getWidth == a.getWidth) |
| |
| out |
| } |
| |
| val slidenb0 = VSliden(0, f2(1,0), MuxOR(slidevn && sz(0), io.read(0).data), MuxOR(slidevn && sz(0), io.read(1).data)) |
| val slidenh0 = VSliden(1, f2(1,0), MuxOR(slidevn && sz(1), io.read(0).data), MuxOR(slidevn && sz(1), io.read(1).data)) |
| val slidenw0 = VSliden(2, f2(1,0), MuxOR(slidevn && sz(2), io.read(0).data), MuxOR(slidevn && sz(2), io.read(1).data)) |
| |
| val slidenb1 = VSliden(0, f2(1,0), MuxOR(slidehn2 && sz(0), io.read(1).data), MuxOR(slidehn2 && sz(0), io.read(2).data)) |
| val slidenh1 = VSliden(1, f2(1,0), MuxOR(slidehn2 && sz(1), io.read(1).data), MuxOR(slidehn2 && sz(1), io.read(2).data)) |
| val slidenw1 = VSliden(2, f2(1,0), MuxOR(slidehn2 && sz(2), io.read(1).data), MuxOR(slidehn2 && sz(2), io.read(2).data)) |
| |
| val slidepb0 = VSlidep(0, f2(1,0), MuxOR(slidevp && sz(0), io.read(0).data), MuxOR(slidevp && sz(0), io.read(1).data)) |
| val slideph0 = VSlidep(1, f2(1,0), MuxOR(slidevp && sz(1), io.read(0).data), MuxOR(slidevp && sz(1), io.read(1).data)) |
| val slidepw0 = VSlidep(2, f2(1,0), MuxOR(slidevp && sz(2), io.read(0).data), MuxOR(slidevp && sz(2), io.read(1).data)) |
| |
| val slidepb1 = VSlidep(0, f2(1,0), MuxOR(slidehp2 && sz(0), io.read(1).data), MuxOR(slidehp2 && sz(0), io.read(2).data)) |
| val slideph1 = VSlidep(1, f2(1,0), MuxOR(slidehp2 && sz(1), io.read(1).data), MuxOR(slidehp2 && sz(1), io.read(2).data)) |
| val slidepw1 = VSlidep(2, f2(1,0), MuxOR(slidehp2 && sz(2), io.read(1).data), MuxOR(slidehp2 && sz(2), io.read(2).data)) |
| |
| val slide0 = slidenb0 | slidenh0 | slidenw0 | |
| slidepb0 | slideph0 | slidepw0 |
| |
| val slide1 = slidenb1 | slidenh1 | slidenw1 | |
| slidepb1 | slideph1 | slidepw1 |
| |
| // --------------------------------------------------------------------------- |
| // Select. |
| def VSel(sz: Int, a: UInt, b: UInt, c: UInt): UInt = { |
| val size = 8 << sz |
| assert(sz == 0 || sz == 1 || sz == 2) |
| assert(size == 8 || size == 16 || size == 32) |
| |
| val cnt = a.getWidth / size |
| val sout = Wire(Vec(cnt, UInt(size.W))) |
| |
| for (i <- 0 until cnt) { |
| val l = i * size // lsb |
| val m = l + size - 1 // msb |
| sout(i) := Mux(a(l), c(m,l), b(m,l)) |
| } |
| |
| val out = sout.asUInt |
| assert(out.getWidth == a.getWidth) |
| |
| out |
| } |
| |
| val selb0 = VSel(0, MuxOR(sel && sz(0), io.read(0).data), MuxOR(sel && sz(0), io.read(1).data), MuxOR(sel && sz(0), io.read(2).data)) |
| val selh0 = VSel(1, MuxOR(sel && sz(1), io.read(0).data), MuxOR(sel && sz(1), io.read(1).data), MuxOR(sel && sz(1), io.read(2).data)) |
| val selw0 = VSel(2, MuxOR(sel && sz(2), io.read(0).data), MuxOR(sel && sz(2), io.read(1).data), MuxOR(sel && sz(2), io.read(2).data)) |
| |
| val sel0 = selb0 | selh0 | selw0 |
| |
| // --------------------------------------------------------------------------- |
| // Even/Odd. |
| def VEvnOdd(sel: Int, sz: Int, a: UInt, b: UInt): UInt = { |
| val size = 8 << sz |
| assert(sz == 0 || sz == 1 || sz == 2) |
| assert(size == 8 || size == 16 || size == 32) |
| assert(sel == 0 || sel == 1) |
| |
| val cnt = a.getWidth / size |
| val h = a.getWidth / 2 |
| val evnodd = Wire(Vec(cnt, UInt(size.W))) |
| |
| for (i <- 0 until cnt / 2) { |
| val j = i * 2 + sel |
| val l = j * size // lsb |
| val m = l + size - 1 // msb |
| evnodd(i) := a(m,l) |
| } |
| |
| for (i <- cnt / 2 until cnt) { |
| val j = (i - cnt / 2) * 2 + sel |
| val l = j * size // lsb |
| val m = l + size - 1 // msb |
| evnodd(i) := b(m,l) |
| } |
| |
| val out = evnodd.asUInt |
| assert(out.getWidth == a.getWidth) |
| |
| out |
| } |
| |
| val evnb = VEvnOdd(0, 0, MuxOR(evn && sz(0), io.read(0).data), MuxOR(evn && sz(0), io.read(1).data)) |
| val evnh = VEvnOdd(0, 1, MuxOR(evn && sz(1), io.read(0).data), MuxOR(evn && sz(1), io.read(1).data)) |
| val evnw = VEvnOdd(0, 2, MuxOR(evn && sz(2), io.read(0).data), MuxOR(evn && sz(2), io.read(1).data)) |
| val oddb = VEvnOdd(1, 0, MuxOR(odd && sz(0), io.read(0).data), MuxOR(odd && sz(0), io.read(1).data)) |
| val oddh = VEvnOdd(1, 1, MuxOR(odd && sz(1), io.read(0).data), MuxOR(odd && sz(1), io.read(1).data)) |
| val oddw = VEvnOdd(1, 2, MuxOR(odd && sz(2), io.read(0).data), MuxOR(odd && sz(2), io.read(1).data)) |
| |
| val evn0 = evnb | evnh | evnw |
| val odd1 = oddb | oddh | oddw |
| |
| // --------------------------------------------------------------------------- |
| // VZip. |
| def VZip(sz: Int, a: UInt, b: UInt): (UInt, UInt) = { |
| val size = 8 << sz |
| assert(sz == 0 || sz == 1 || sz == 2) |
| assert(size == 8 || size == 16 || size == 32) |
| |
| val cnt = a.getWidth / size |
| val h = a.getWidth / 2 |
| val zip0 = Wire(Vec(cnt, UInt(size.W))) |
| val zip1 = Wire(Vec(cnt, UInt(size.W))) |
| |
| for (i <- 0 until cnt) { |
| val j = i / 2 |
| val l = j * size // lsb |
| val m = l + size - 1 // msb |
| if ((i & 1) == 0) { |
| zip0(i) := a(m+0,l+0) |
| zip1(i) := a(m+h,l+h) |
| } else { |
| zip0(i) := b(m+0,l+0) |
| zip1(i) := b(m+h,l+h) |
| } |
| } |
| |
| val out0 = zip0.asUInt |
| val out1 = zip1.asUInt |
| assert(out0.getWidth == a.getWidth) |
| assert(out1.getWidth == a.getWidth) |
| |
| (out0, out1) |
| } |
| |
| val (zipb0, zipb1) = VZip(0, MuxOR(zip && sz(0), io.read(0).data), MuxOR(zip && sz(0), io.read(1).data)) |
| val (ziph0, ziph1) = VZip(1, MuxOR(zip && sz(1), io.read(0).data), MuxOR(zip && sz(1), io.read(1).data)) |
| val (zipw0, zipw1) = VZip(2, MuxOR(zip && sz(2), io.read(0).data), MuxOR(zip && sz(2), io.read(1).data)) |
| |
| val zip0 = zipb0 | ziph0 | zipw0 |
| val zip1 = zipb1 | ziph1 | zipw1 |
| |
| // --------------------------------------------------------------------------- |
| // Depthwise. |
| val (dwconv0, dwconv1) = |
| if (aluid == 0) { |
| VDot(aluid, dwconv, |
| VecInit(io.read(0).data, io.read(1).data, io.read(2).data), |
| VecInit(io.read(3).data, io.read(4).data, io.read(5).data), sv) |
| } else { |
| VDot(aluid, dwconv, |
| VecInit(io.read(3).data, io.read(4).data, io.read(5).data), |
| VecInit(io.read(0).data, io.read(1).data, io.read(2).data), sv) |
| } |
| |
| // --------------------------------------------------------------------------- |
| // Parallel Load registered VAluIntLane stage. |
| load(0) := evn0 | zip0 | slide0 | dwconv0 | sel0 |
| load(1) := odd1 | zip1 | slide1 | dwconv1 |
| |
| // --------------------------------------------------------------------------- |
| // Outputs. |
| val vddata = Wire(Vec(lanes, UInt(32.W))) |
| val vedata = Wire(Vec(lanes, UInt(32.W))) |
| |
| for (i <- 0 until lanes) { |
| vddata(i) := valu(i).io.write(0).data |
| vedata(i) := valu(i).io.write(1).data |
| } |
| |
| io.write(0).valid := vdvalid1 |
| io.write(0).addr := vdaddr1.addr |
| io.write(0).data := vddata.asUInt |
| |
| io.write(1).valid := vevalid1 |
| io.write(1).addr := veaddr1.addr |
| io.write(1).data := vedata.asUInt |
| |
| io.whint(0).valid := vdvalid0 && !wmask |
| io.whint(0).addr := vdaddr0.addr |
| |
| io.whint(1).valid := vevalid0 && !wmask |
| io.whint(1).addr := veaddr0.addr |
| } |
| |
| class VAluIntLane extends Module { |
| val e = new VEncodeOp() |
| |
| val io = IO(new Bundle { |
| val in = Input(new Bundle { |
| val vdvalid = Bool() |
| val vevalid = Bool() |
| val sz = UInt(3.W) |
| val negative = Bool() |
| val round = Bool() |
| val signed = Bool() |
| }) |
| val op = Input(new Bundle { |
| val absd = Bool() |
| val acc = Bool() |
| val dup = Bool() |
| val max = Bool() |
| val min = Bool() |
| val mv = Bool() |
| val mv2 = Bool() |
| val mvp = Bool() |
| val srans = Bool() |
| val sraqs = Bool() |
| |
| val dwinit = Bool() |
| val dwconv = Bool() |
| val dwconvData = Bool() |
| |
| val add = new Bundle { |
| val en = Bool() |
| val add = Bool() |
| val adds = Bool() |
| val addw = Bool() |
| val add3 = Bool() |
| val hadd = Bool() |
| } |
| |
| val cmp = new Bundle { |
| val en = Bool() |
| val eq = Bool() |
| val ne = Bool() |
| val lt = Bool() |
| val le = Bool() |
| val gt = Bool() |
| val ge = Bool() |
| } |
| |
| val log = new Bundle { |
| val en = Bool() |
| val and = Bool() |
| val or = Bool() |
| val xor = Bool() |
| val not = Bool() |
| val rev = Bool() |
| val ror = Bool() |
| val clb = Bool() |
| val clz = Bool() |
| val cpop = Bool() |
| } |
| |
| val mul0 = new Bundle { |
| val en = Bool() |
| val dmulh = Bool() |
| val mul = Bool() |
| val mulh = Bool() |
| val muls = Bool() |
| val mulw = Bool() |
| val madd = Bool() |
| } |
| |
| val mul1 = new Bundle { |
| val en = Bool() |
| val dmulh = Bool() |
| val mul = Bool() |
| val mulh = Bool() |
| val muls = Bool() |
| } |
| |
| val padd = new Bundle { |
| val en = Bool() |
| val add = Bool() |
| val sub = Bool() |
| } |
| |
| val rsub = new Bundle { |
| val en = Bool() |
| val rsub = Bool() |
| } |
| |
| val shf = new Bundle { |
| val en = new Bundle { |
| val l = Bool() // left |
| val r = Bool() // right |
| } |
| val shl = Bool() |
| val shr = Bool() |
| val shf = Bool() |
| } |
| |
| val sub = new Bundle { |
| val en = Bool() |
| val sub = Bool() |
| val subs = Bool() |
| val subw = Bool() |
| val hsub = Bool() |
| } |
| }) |
| val read = Vec(7, Input(new Bundle { |
| val data = UInt(32.W) |
| })) |
| val write = Vec(2, Output(new Bundle { |
| val data = UInt(32.W) |
| })) |
| val load = Vec(2, Input(UInt(32.W))) // parallel load data |
| }) |
| |
| def VAlu(sz: Int, a: UInt, b: UInt, c: UInt, d: UInt, e: UInt, f: UInt): (UInt, UInt, UInt, UInt, UInt, UInt) = { |
| // Note: sz is source size, not destination as is ISA defined. |
| val size = 8 << sz |
| assert(sz == 0 || sz == 1 || sz == 2) |
| assert(size == 8 || size == 16 || size == 32) |
| assert(a.getWidth == b.getWidth) |
| assert(a.getWidth == c.getWidth) |
| assert(a.getWidth == 32) |
| val cnt = a.getWidth / size |
| val alu0 = Wire(Vec(cnt, UInt(size.W))) |
| val alu1 = Wire(Vec(cnt, UInt(size.W))) |
| val aluw0 = Wire(Vec(cnt / 2, UInt((2 * size).W))) |
| val aluw1 = Wire(Vec(cnt / 2, UInt((2 * size).W))) |
| val rnd0 = Wire(Vec(cnt, UInt(size.W))) |
| val rnd1 = Wire(Vec(cnt, UInt(size.W))) |
| |
| // ------------------------------------------------------------------------- |
| // Controls. |
| val negative = io.in.negative |
| val round = io.in.round |
| val signed = io.in.signed |
| |
| // ------------------------------------------------------------------------- |
| // Datapath. |
| val aw = a |
| val bw = b |
| val cw = c |
| val dw = d |
| val ew = e |
| val fw = f |
| |
| val acc_a = MuxOR(io.op.acc, aw) |
| val acc_b = MuxOR(io.op.acc, bw) |
| val acc_c = MuxOR(io.op.acc, cw) |
| |
| val add_a = MuxOR(io.op.add.en, aw) |
| val add_b = MuxOR(io.op.add.en, bw) |
| val add_r = io.op.add.hadd && round |
| |
| val cmp_a = MuxOR(io.op.cmp.en, aw) |
| val cmp_b = MuxOR(io.op.cmp.en, bw) |
| |
| val log_a = MuxOR(io.op.log.en, aw) |
| val log_b = MuxOR(io.op.log.en, bw) |
| |
| val mul0_a = MuxOR(io.op.mul0.en, aw) |
| val mul0_b = MuxOR(io.op.mul0.en, bw) |
| val mul1_a = MuxOR(io.op.mul1.en, cw) |
| val mul1_b = MuxOR(io.op.mul1.en, bw) |
| |
| val padd_a = MuxOR(io.op.padd.en, aw) |
| |
| val rsub_a = MuxOR(io.op.rsub.en, aw) |
| val rsub_b = MuxOR(io.op.rsub.en, bw) |
| |
| val shl_a = MuxOR(io.op.shf.en.l, aw) |
| val shl_b = MuxOR(io.op.shf.en.l, bw) |
| val shr_a = MuxOR(io.op.shf.en.r, aw) |
| val shr_b = MuxOR(io.op.shf.en.r, bw) |
| |
| val srans_a = MuxOR(io.op.srans, aw) |
| val srans_b = MuxOR(io.op.srans, bw) |
| val srans_c = MuxOR(io.op.srans, cw) |
| |
| val sraqs_a = MuxOR(io.op.sraqs, aw) |
| val sraqs_b = MuxOR(io.op.sraqs, bw) |
| val sraqs_c = MuxOR(io.op.sraqs, cw) |
| val sraqs_d = MuxOR(io.op.sraqs, dw) |
| val sraqs_f = MuxOR(io.op.sraqs, fw) |
| |
| val sub_a = MuxOR(io.op.sub.en, aw) |
| val sub_b = MuxOR(io.op.sub.en, bw) |
| val sub_r = io.op.sub.hsub && round |
| |
| // ------------------------------------------------------------------------- |
| // Functions. |
| for (i <- 0 until cnt) { |
| val l = i * size // lsb |
| val m = l + size - 1 // msb |
| val ln = (i / 2) * 2 * size // lsb narrowing |
| val mn = ln + 2 * size - 1 // msb narrowing |
| val lq = (i / 4) * 4 * size // lsb narrowing |
| val mq = lq + 4 * size - 1 // msb narrowing |
| val mshamt = l + log2Ceil(size) - 1 |
| |
| // ----------------------------------------------------------------------- |
| // Arithmetic. |
| val add_sa = add_a(m) && signed |
| val add_sb = add_b(m) && signed |
| val adder = (Cat(add_sa, add_a(m,l)).asSInt +& Cat(add_sb, add_b(m,l)).asSInt).asUInt + add_r |
| val sataddmsb = adder(size, size - 1) |
| val sataddsel = |
| Cat( signed && sataddmsb === 2.U, // vadd.s -ve |
| signed && sataddmsb === 1.U, // vadd.s +ve |
| !signed && sataddmsb(1)) // vadd.su +ve |
| assert(PopCount(sataddsel) <= 1.U) |
| |
| val sub_sa = sub_a(m) && signed |
| val sub_sb = sub_b(m) && signed |
| val subtr = (Cat(sub_sa, sub_a(m,l)).asSInt -& Cat(sub_sb, sub_b(m,l)).asSInt).asUInt + sub_r |
| val satsubmsb = subtr(size, size - 1) |
| val satsubsel = |
| Cat( signed && satsubmsb === 2.U, // vsub.s -ve |
| signed && satsubmsb === 1.U, // vsub.s +ve |
| !signed && satsubmsb(1)) // vsub.su 0 |
| assert(PopCount(satsubsel) <= 1.U) |
| |
| val rsubtr = rsub_b(m,l) - rsub_a(m,l) |
| |
| val xeq = cmp_a(m,l) === cmp_b(m,l) |
| val xne = cmp_a(m,l) =/= cmp_b(m,l) |
| val slt = cmp_a(m,l).asSInt < cmp_b(m,l).asSInt |
| val ult = cmp_a(m,l) < cmp_b(m,l) |
| val sle = slt || xeq |
| val ule = ult || xeq |
| |
| val sult = Mux(signed, slt, ult) |
| |
| def Shift(a: UInt, b: UInt, sln: UInt, sra: UInt, srl: UInt): UInt = { |
| assert(a.getWidth == size) |
| assert(b.getWidth == size) |
| assert(sln.getWidth == (2 * size - 1)) |
| assert(sra.getWidth == size) |
| assert(srl.getWidth == size) |
| val slnsz = sln(size - 1, 0) |
| val input_neg = a(size - 1) |
| val input_zero = a === 0.U |
| val shamt_neg = b(size - 1) |
| val rs = Wire(UInt(size.W)) |
| val ru = Wire(UInt(size.W)) |
| if (true) { |
| val shamt_negsat = b.asSInt <= (-(size - 1)).S |
| val shamt_possat = b.asSInt >= (size - 1).S |
| val signb = ~0.U(size.W) >> (b(log2Ceil(size) - 1, 0) - 1.U) |
| val possat = shamt_neg && !input_neg && (shamt_negsat || (sln(2 * size - 2, size - 1) =/= 0.U )) && !input_zero |
| val negsat = shamt_neg && input_neg && (shamt_negsat || (sln(2 * size - 2, size - 1) =/= signb)) |
| assert(!(possat && negsat)) |
| val posmax = Cat(0.U(1.W), ~0.U((size - 1).W)) |
| val negmin = Cat(1.U(1.W), 0.U((size - 1).W)) |
| assert(posmax.getWidth == size) |
| assert(negmin.getWidth == size) |
| |
| rs := MuxOR(!shamt_neg && !shamt_possat, sra) | |
| MuxOR(!shamt_neg && shamt_possat && input_neg, ~0.U(size.W)) | |
| MuxOR( shamt_neg && !possat && !negsat, slnsz) | |
| MuxOR( shamt_neg && possat, posmax) | |
| MuxOR( shamt_neg && negsat, negmin) |
| } |
| if (true) { |
| val shamt_negsat = b.asSInt <= -size.S |
| val shamt_possat = b.asSInt >= size.S |
| val possat = shamt_neg && (shamt_negsat || (sln(2 * size - 2, size) =/= 0.U)) && !input_zero |
| val posmax = ~0.U(size.W) |
| assert(posmax.getWidth == size) |
| |
| ru := MuxOR(!shamt_neg && !shamt_possat, srl) | |
| MuxOR( shamt_neg && !possat, slnsz) | |
| MuxOR( shamt_neg && possat, posmax) |
| } |
| Mux(signed, rs, ru) |
| } |
| |
| def Round(a: UInt, b: UInt): UInt = { |
| assert(a.getWidth == size) |
| assert(b.getWidth == size) |
| val input_neg = a(size - 1) |
| val shamt_neg = b(size - 1) |
| val shamt_zero = b === 0.U |
| val rbit = Cat(a(size - 2, 0), a(size - 1))(b(log2Ceil(size) - 1, 0)) // shf: idx[8] == idx[0] |
| val shamt_possat = Mux(signed, b.asSInt >= size.S, b.asSInt > size.S) |
| val r = MuxOR(round && !shamt_possat && !shamt_neg && !shamt_zero, rbit) | |
| MuxOR(round && shamt_possat && input_neg && signed, 1.U) |
| assert(r.getWidth == 1) |
| r |
| } |
| |
| val shl = (shl_a(m,l) << shl_b(mshamt, l))(size - 1, 0) |
| val sln = (shl_a(m,l) << (size.U - shl_b(mshamt, l)))(2 * size - 2, 0) |
| val srl = shr_a(m,l) >> shr_b(mshamt, l) |
| val srs = MuxOR(shr_a(m), ((~0.U(size.W)) << ((size - 1).U - shr_b(mshamt, l)))(size - 1, 0)) |
| val sra = srs | srl |
| val shf = Shift(shl_a(m,l), shl_b(m,l), sln, sra, srl) |
| val shr = Mux(signed, sra, srl) |
| assert(shl.getWidth == size) |
| assert(sln.getWidth == (2 * size - 1)) |
| assert(sra.getWidth == size) |
| assert(srl.getWidth == size) |
| assert(srs.getWidth == size) |
| assert(shf.getWidth == size) |
| |
| val shf_rnd = Round(shl_a(m,l), shl_b(m,l)) |
| assert(shf_rnd.getWidth == 1) |
| |
| def Srans(s: Int, a: UInt, b: UInt): UInt = { |
| assert(s == 2 || s == 4) |
| assert(a.getWidth == size * s) |
| assert(b.getWidth == size) |
| |
| val shamt = b(log2Ceil(s * size) - 1, 0) |
| val srl = a >> shamt |
| // Signed MSB padding for negative input a. Otherwise it should always |
| // pad with zeros. |
| val srs = MuxOR(a(s * size - 1) && signed, |
| ((~0.U((s * size).W)) << ((s * size - 1).U - shamt))(s * size - 1, 0)) |
| val sra = srs | srl |
| assert(srl.getWidth == (s * size)) |
| assert(srs.getWidth == (s * size)) |
| val rbit = Cat(a(s * size - 2, 0), 0.U(1.W))(shamt) |
| assert(rbit.getWidth == 1) |
| |
| val umax = ((1 << size) - 1).U((s * size).W) |
| val smax = ((1 << (size - 1)) - 1).S((s * size).W) |
| val smin = -(1 << (size - 1)).S((s * size).W) |
| val rshf = Mux(round && rbit, sra + 1.U, sra) |
| |
| val is_umax = !signed && (rshf.asUInt > umax) |
| // No unsigned negative capping because it's always >=0. |
| val is_smax = signed && (rshf.asSInt > smax) |
| val is_smin = signed && (rshf.asSInt < smin) |
| val is_norm = !(is_umax || is_smax || is_smin) |
| assert(PopCount(Cat(is_umax, is_smax, is_smin, is_norm)) <= 1.U) |
| |
| val r = MuxOR(is_umax, umax.asUInt(size - 1, 0)) | |
| MuxOR(is_smax, smax.asUInt(size - 1, 0)) | |
| MuxOR(is_smin, smin.asUInt(size - 1, 0)) | |
| MuxOR(is_norm, rshf(size - 1, 0)) |
| assert(r.getWidth == size) |
| r |
| } |
| |
| def Rev(a: UInt, s: UInt): UInt = { |
| if (size == 32) { |
| val b = Mux(!s(0), a, Cat(a(30), a(31), a(28), a(29), a(26), a(27), a(24), a(25), |
| a(22), a(23), a(20), a(21), a(18), a(19), a(16), a(17), |
| a(14), a(15), a(12), a(13), a(10), a(11), a( 8), a( 9), |
| a( 6), a( 7), a( 4), a( 5), a( 2), a( 3), a( 0), a( 1))) |
| val c = Mux(!s(1), b, Cat(b(29,28), b(31,30), b(25,24), b(27,26), |
| b(21,20), b(23,22), b(17,16), b(19,18), |
| b(13,12), b(15,14), b( 9, 8), b(11,10), |
| b( 5, 4), b( 7, 6), b( 1, 0), b( 3, 2))) |
| val d = Mux(!s(2), c, Cat(c(27,24), c(31,28), c(19,16), c(23,20), |
| c(11, 8), c(15,12), c( 3, 0), c( 7, 4))) |
| val e = Mux(!s(3), d, Cat(d(23,16), d(31,24), d( 7, 0), d(15, 8))) |
| val f = Mux(!s(4), e, Cat(e(15, 0), e(31,16))) |
| assert(a.getWidth == 32) |
| assert(b.getWidth == 32) |
| assert(c.getWidth == 32) |
| assert(d.getWidth == 32) |
| assert(e.getWidth == 32) |
| assert(f.getWidth == 32) |
| f |
| } else if (size == 16) { |
| val b = Mux(!s(0), a, Cat(a(14), a(15), a(12), a(13), a(10), a(11), a( 8), a( 9), |
| a( 6), a( 7), a( 4), a( 5), a( 2), a( 3), a( 0), a( 1))) |
| val c = Mux(!s(1), b, Cat(b(13,12), b(15,14), b( 9, 8), b(11,10), |
| b( 5, 4), b( 7, 6), b( 1, 0), b( 3, 2))) |
| val d = Mux(!s(2), c, Cat(c(11, 8), c(15,12), c( 3, 0), c( 7, 4))) |
| val e = Mux(!s(3), d, Cat(d( 7, 0), d(15, 8))) |
| assert(a.getWidth == 16) |
| assert(b.getWidth == 16) |
| assert(c.getWidth == 16) |
| assert(d.getWidth == 16) |
| assert(e.getWidth == 16) |
| e |
| } else { |
| val b = Mux(!s(0), a, Cat(a(6), a(7), a(4), a(5), a(2), a(3), a(0), a(1))) |
| val c = Mux(!s(1), b, Cat(b(5, 4), b(7, 6), b(1, 0), b( 3, 2))) |
| val d = Mux(!s(2), c, Cat(c(3, 0), c(7, 4))) |
| assert(a.getWidth == 8) |
| assert(b.getWidth == 8) |
| assert(c.getWidth == 8) |
| assert(d.getWidth == 8) |
| d |
| } |
| } |
| |
| def Ror(a: UInt, s: UInt): UInt = { |
| if (size == 32) { |
| val b = Mux(!s(0), a, Cat(a(0), a(31,1))) |
| val c = Mux(!s(1), b, Cat(b(1,0), b(31,2))) |
| val d = Mux(!s(2), c, Cat(c(3,0), c(31,4))) |
| val e = Mux(!s(3), d, Cat(d(7,0), d(31,8))) |
| val f = Mux(!s(4), e, Cat(e(15,0), e(31,16))) |
| assert(a.getWidth == 32) |
| assert(b.getWidth == 32) |
| assert(c.getWidth == 32) |
| assert(d.getWidth == 32) |
| assert(e.getWidth == 32) |
| assert(f.getWidth == 32) |
| f |
| } else if (size == 16) { |
| val b = Mux(!s(0), a, Cat(a(0), a(15,1))) |
| val c = Mux(!s(1), b, Cat(b(1,0), b(15,2))) |
| val d = Mux(!s(2), c, Cat(c(3,0), c(15,4))) |
| val e = Mux(!s(3), d, Cat(d(7,0), d(15,8))) |
| assert(a.getWidth == 16) |
| assert(b.getWidth == 16) |
| assert(c.getWidth == 16) |
| assert(d.getWidth == 16) |
| assert(e.getWidth == 16) |
| e |
| } else { |
| val b = Mux(!s(0), a, Cat(a(0), a(7,1))) |
| val c = Mux(!s(1), b, Cat(b(1,0), b(7,2))) |
| val d = Mux(!s(2), c, Cat(c(3,0), c(7,4))) |
| assert(a.getWidth == 8) |
| assert(b.getWidth == 8) |
| assert(c.getWidth == 8) |
| assert(d.getWidth == 8) |
| d |
| } |
| } |
| |
| val mul0_as = Cat(signed && mul0_a(m), mul0_a(m,l)) |
| val mul0_bs = Cat(signed && mul0_b(m), mul0_b(m,l)) |
| val mul0_sign = mul0_a(m) =/= mul0_b(m) && mul0_a(m,l) =/= 0.U && mul0_b(m,l) =/= 0.U |
| val prod0 = (mul0_as.asSInt * mul0_bs.asSInt).asUInt |
| val prodh0 = prod0(2 * size - 1, size) |
| val proddh0 = prod0(2 * size - 2, size - 1) |
| |
| val mul1_as = Cat(signed && mul1_a(m), mul1_a(m,l)) |
| val mul1_bs = Cat(signed && mul1_b(m), mul1_b(m,l)) |
| val mul1_sign = mul1_a(m) =/= mul1_b(m) && mul1_a(m,l) =/= 0.U && mul1_b(m,l) =/= 0.U |
| val prod1 = (mul1_as.asSInt * mul1_bs.asSInt).asUInt |
| val prodh1 = prod1(2 * size - 1, size) |
| val proddh1 = prod1(2 * size - 2, size - 1) |
| |
| val muls0_umax = !signed && prodh0 =/= 0.U |
| val muls0_smax = signed && !mul0_sign && ( prod0(size - 1) || prodh0 =/= 0.U(size.W)) |
| val muls0_smin = signed && mul0_sign && (!prod0(size - 1) || prodh0 =/= ~0.U(size.W)) |
| val muls0_base = !(muls0_umax || muls0_smax || muls0_smin) |
| assert(PopCount(Cat(muls0_umax, muls0_smax, muls0_smin, muls0_base)) <= 1.U) |
| |
| val muls1_umax = !signed && prodh1 =/= 0.U |
| val muls1_smax = signed && !mul1_sign && ( prod1(size - 1) || prodh1 =/= 0.U(size.W)) |
| val muls1_smin = signed && mul1_sign && (!prod1(size - 1) || prodh1 =/= ~0.U(size.W)) |
| val muls1_base = !(muls1_umax || muls1_smax || muls1_smin) |
| assert(PopCount(Cat(muls1_umax, muls1_smax, muls1_smin, muls1_base)) <= 1.U) |
| |
| val maxneg = Cat(1.U(1.W), 0.U((size - 1).W)) // 0x80... |
| |
| val dmulh0_possat = mul0_a(m,l) === maxneg && mul0_b(m,l) === maxneg |
| |
| val dmulh1_possat = mul1_a(m,l) === maxneg && mul1_b(m,l) === maxneg |
| |
| val dmulh0 = MuxOR(!dmulh0_possat, proddh0) | |
| MuxOR(dmulh0_possat, Cat(0.U(1.W), ~0.U((size - 1).W))) // 0x7f... |
| |
| val dmulh1 = MuxOR(!dmulh1_possat, proddh1) | |
| MuxOR(dmulh1_possat, Cat(0.U(1.W), ~0.U((size - 1).W))) // 0x7f... |
| |
| val mulh0 = prodh0 |
| val mulh1 = prodh1 |
| |
| val muls0 = MuxOR(muls0_umax, ~0.U(size.W)) | |
| MuxOR(muls0_smax, ~0.U((size - 1).W)) | |
| MuxOR(muls0_smin, Cat(1.U(1.W), 0.U((size - 1).W))) | |
| MuxOR(muls0_base, prod0(size - 1, 0)) |
| |
| val muls1 = MuxOR(muls1_umax, ~0.U(size.W)) | |
| MuxOR(muls1_smax, ~0.U((size - 1).W)) | |
| MuxOR(muls1_smin, Cat(1.U(1.W), 0.U((size - 1).W))) | |
| MuxOR(muls1_base, prod1(size - 1, 0)) |
| |
| val dmulh0_rnd = MuxOR(round && io.op.mul0.dmulh && io.in.sz(sz) && !dmulh0_possat, |
| Mux(negative && mul0_sign, |
| MuxOR(!prod0(size - 2), ~0.U(size.W)), // -1 |
| MuxOR( prod0(size - 2), 1.U(size.W)))) // +1 |
| |
| val dmulh1_rnd = MuxOR(round && io.op.mul1.dmulh && io.in.sz(sz) && !dmulh1_possat, |
| Mux(negative && mul1_sign, |
| MuxOR(!prod1(size - 2), ~0.U(size.W)), // -1 |
| MuxOR( prod1(size - 2), 1.U(size.W)))) // +1 |
| |
| val mulh0_rnd = round && io.op.mul0.mulh && prod0(size - 1) |
| val mulh1_rnd = round && io.op.mul1.mulh && prod1(size - 1) |
| |
| // ----------------------------------------------------------------------- |
| // Operations. |
| val absd = MuxOR(io.op.absd, Mux(sult, rsubtr, subtr(size - 1, 0))) |
| assert(absd.getWidth == size) |
| |
| val acc = if (sz == 0 || sz == 1) { // size / 2 |
| if ((i & 1) == 0) { |
| acc_a(mn,ln) + SignExt(Cat(signed & acc_b(m), acc_b(m,l)), 2 * size) |
| } else { |
| acc_c(mn,ln) + SignExt(Cat(signed & acc_b(m), acc_b(m,l)), 2 * size) |
| } |
| } else { |
| 0.U((2 * size).W) |
| } |
| assert(acc.getWidth == (2 * size)) |
| |
| val add = MuxOR(sataddsel(2) && io.op.add.adds, Cat(1.U(1.W), 0.U((size - 1).W))) | |
| MuxOR(sataddsel(1) && io.op.add.adds, ~0.U((size - 1).W)) | |
| MuxOR(sataddsel(0) && io.op.add.adds, ~0.U(size.W)) | |
| MuxOR(sataddsel === 0.U && io.op.add.adds || io.op.add.add || io.op.add.add3, adder(size - 1, 0)) | |
| MuxOR(io.op.add.hadd, adder(size, 1)) |
| |
| val addw = MuxOR(io.op.add.addw, SignExt(adder, 2 * size)) |
| assert(addw.getWidth == (2 * size)) |
| |
| val dup = MuxOR(io.op.dup, io.read(1).data(m,l)) |
| |
| val max = MuxOR(io.op.max, Mux(sult, cmp_b(m,l), cmp_a(m,l))) |
| val min = MuxOR(io.op.min, Mux(sult, cmp_a(m,l), cmp_b(m,l))) |
| |
| val mul0 = MuxOR(io.op.mul0.mul || io.op.mul0.madd, prod0(size - 1, 0)) | |
| MuxOR(io.op.mul0.dmulh, dmulh0) | |
| MuxOR(io.op.mul0.mulh, mulh0) | |
| MuxOR(io.op.mul0.muls, muls0) |
| |
| val mul1 = MuxOR(io.op.mul1.mul, prod1(size - 1, 0)) | |
| MuxOR(io.op.mul1.dmulh, dmulh1) | |
| MuxOR(io.op.mul1.mulh, mulh1) | |
| MuxOR(io.op.mul1.muls, muls1) |
| |
| val mulw = MuxOR(io.op.mul0.mulw, prod0(2 * size - 1, 0)) |
| |
| val padd = |
| if (sz == 1 || sz == 2) { |
| val p0 = i * size |
| val p1 = p0 + size / 2 - 1 |
| val p2 = p1 + 1 |
| val p3 = p0 + size - 1 |
| val a = Cat(signed && padd_a(p1), padd_a(p1,p0)) |
| val b = Cat(signed && padd_a(p3), padd_a(p3,p2)) |
| val add = MuxOR(io.op.padd.add, SignExt((a.asSInt +& b.asSInt).asUInt, size)) |
| val sub = MuxOR(io.op.padd.sub, SignExt((a.asSInt -& b.asSInt).asUInt, size)) |
| assert(add.getWidth == size) |
| assert(sub.getWidth == size) |
| add | sub |
| } else { |
| 0.U(size.W) |
| } |
| |
| val rsub = MuxOR(io.op.rsub.rsub, rsubtr) |
| |
| val srans = if (sz == 0 || sz == 1) { // size / 2 |
| if ((i & 1) == 0) { |
| Srans(2, srans_a(mn,ln), srans_b(m,l)) |
| } else { |
| Srans(2, srans_c(mn,ln), srans_b(m,l)) |
| } |
| } else { |
| 0.U(size.W) |
| } |
| |
| val sraqs = if (sz == 0) { // size / 4 |
| if ((i & 3) == 0) { |
| Srans(4, sraqs_a(mq,lq), sraqs_b(m,l)) |
| } else if ((i & 3) == 1) { |
| Srans(4, sraqs_d(mq,lq), sraqs_b(m,l)) |
| } else if ((i & 3) == 2) { |
| Srans(4, sraqs_c(mq,lq), sraqs_b(m,l)) |
| } else { |
| Srans(4, sraqs_f(mq,lq), sraqs_b(m,l)) |
| } |
| } else { |
| 0.U(size.W) |
| } |
| |
| val sub = MuxOR(satsubsel(2) && io.op.sub.subs, Cat(1.U(1.W), 0.U((size - 1).W))) | |
| MuxOR(satsubsel(1) && io.op.sub.subs, ~0.U((size - 1).W)) | |
| MuxOR(satsubsel(0) && io.op.sub.subs, 0.U(size.W)) | |
| MuxOR(satsubsel === 0.U && io.op.sub.subs || io.op.sub.sub, subtr(size - 1, 0)) | |
| MuxOR(io.op.sub.hsub, subtr(size, 1)) |
| |
| val subw = MuxOR(io.op.sub.subw, SignExt(subtr, 2 * size)) |
| assert(subw.getWidth == (2 * size)) |
| |
| val cmp = io.in.sz(sz) && |
| (MuxOR(io.op.cmp.eq, xeq) | |
| MuxOR(io.op.cmp.ne, xne) | |
| MuxOR(io.op.cmp.lt && signed, slt) | |
| MuxOR(io.op.cmp.lt && !signed, ult) | |
| MuxOR(io.op.cmp.le && signed, sle) | |
| MuxOR(io.op.cmp.le && !signed, ule) | |
| MuxOR(io.op.cmp.gt && signed, !sle) | |
| MuxOR(io.op.cmp.gt && !signed, !ule) | |
| MuxOR(io.op.cmp.ge && signed, !slt) | |
| MuxOR(io.op.cmp.ge && !signed, !ult)) |
| assert(cmp.getWidth == 1) |
| |
| val log = |
| MuxOR(io.op.log.and, log_a(m,l) & log_b(m,l)) | |
| MuxOR(io.op.log.or, log_a(m,l) | log_b(m,l)) | |
| MuxOR(io.op.log.xor, log_a(m,l) ^ log_b(m,l)) | |
| MuxOR(io.op.log.not, MuxOR(io.in.sz(sz), ~log_a(m,l))) | |
| MuxOR(io.op.log.rev, Rev(log_a(m,l), log_b(m,l))) | |
| MuxOR(io.op.log.ror, MuxOR(io.in.sz(sz), Ror(log_a(m,l), log_b(m,l)))) | |
| MuxOR(io.op.log.clb, MuxOR(io.in.sz(sz), Clb(log_a(m,l)))) | |
| MuxOR(io.op.log.clz, MuxOR(io.in.sz(sz), Clz(log_a(m,l)))) | |
| MuxOR(io.op.log.cpop, PopCount(log_a(m,l))) |
| assert(log.getWidth == size) |
| |
| val shift = |
| MuxOR(io.op.shf.shl, shl) | |
| MuxOR(io.op.shf.shr, shr) | |
| MuxOR(io.op.shf.shf, shf) |
| assert(shf.getWidth == size) |
| |
| val alu_oh = Cat(absd =/= 0.U, |
| add =/= 0.U, |
| cmp =/= 0.U, |
| dup =/= 0.U, |
| log =/= 0.U, |
| max =/= 0.U, |
| min =/= 0.U, |
| mul0 =/= 0.U, |
| padd =/= 0.U, |
| rsub =/= 0.U, |
| shift =/= 0.U, |
| srans =/= 0.U, |
| sraqs =/= 0.U, |
| sub =/= 0.U) |
| |
| assert(PopCount(alu_oh) <= 1.U) |
| |
| alu0(i) := mul0 | absd | add | cmp | dup | log | max | min | padd | rsub | shift | srans | sraqs | sub | |
| MuxOR(io.op.mv, aw(m,l)) |
| |
| alu1(i) := mul1 | |
| MuxOR(io.op.mvp, bw(m,l)) | |
| MuxOR(io.op.mv2, cw(m,l)) |
| |
| rnd0(i) := dmulh0_rnd | mulh0_rnd | shf_rnd |
| rnd1(i) := dmulh1_rnd | mulh1_rnd |
| |
| if (sz < 2) { |
| if ((i & 1) == 0) { |
| aluw0(i / 2) := acc | addw | mulw | subw |
| } else { |
| aluw1(i / 2) := acc | addw | mulw | subw |
| } |
| } |
| } |
| |
| val out_alu0 = alu0.asUInt |
| val out_alu1 = alu1.asUInt |
| val out_rnd0 = rnd0.asUInt |
| val out_rnd1 = rnd1.asUInt |
| val out_aluw0 = aluw0.asUInt |
| val out_aluw1 = aluw1.asUInt |
| assert(out_alu0.getWidth == a.getWidth) |
| assert(out_alu1.getWidth == a.getWidth) |
| assert(out_rnd0.getWidth == a.getWidth) |
| if (sz < 2) { |
| assert(out_aluw0.getWidth == a.getWidth) |
| assert(out_aluw1.getWidth == a.getWidth) |
| } |
| |
| (out_alu0, out_alu1, out_rnd0, out_rnd1, out_aluw0, out_aluw1) |
| } |
| |
| // --------------------------------------------------------------------------- |
| // Data mux. |
| val ina_b = MuxOR(io.in.sz(0), io.read(0).data) |
| val inb_b = MuxOR(io.in.sz(0), io.read(1).data) |
| val inc_b = MuxOR(io.in.sz(0), io.read(2).data) |
| val ind_b = MuxOR(io.in.sz(0), io.read(3).data) |
| val ine_b = MuxOR(io.in.sz(0), io.read(4).data) |
| val inf_b = MuxOR(io.in.sz(0), io.read(5).data) |
| |
| val ina_h = MuxOR(io.in.sz(1), io.read(0).data) |
| val inb_h = MuxOR(io.in.sz(1), io.read(1).data) |
| val inc_h = MuxOR(io.in.sz(1), io.read(2).data) |
| val ind_h = MuxOR(io.in.sz(1), io.read(4).data) |
| val ine_h = MuxOR(io.in.sz(1), io.read(5).data) |
| val inf_h = MuxOR(io.in.sz(1), io.read(6).data) |
| |
| val ina_w = MuxOR(io.in.sz(2), io.read(0).data) |
| val inb_w = MuxOR(io.in.sz(2), io.read(1).data) |
| val inc_w = MuxOR(io.in.sz(2), io.read(2).data) |
| val ind_w = MuxOR(io.in.sz(2), io.read(3).data) |
| val ine_w = MuxOR(io.in.sz(2), io.read(4).data) |
| val inf_w = MuxOR(io.in.sz(2), io.read(5).data) |
| |
| val (outb0, outb1, rndb0, rndb1, outwb0, outwb1) = VAlu(0, ina_b, inb_b, inc_b, ind_b, ine_b, inf_b) |
| val (outh0, outh1, rndh0, rndh1, outwh0, outwh1) = VAlu(1, ina_h, inb_h, inc_h, ind_h, ine_h, inf_h) |
| val (outw0, outw1, rndw0, rndw1, _, _) = VAlu(2, ina_w, inb_w, inc_w, ind_w, ine_w, inf_w) |
| |
| val out0 = outb0 | outh0 | outw0 | outwb0 | outwh0 |
| val out1 = outb1 | outh1 | outw1 | outwb1 | outwh1 |
| val rnd0 = rndb0 | rndh0 | rndw0 |
| val rnd1 = rndb1 | rndh1 | rndw1 |
| |
| // --------------------------------------------------------------------------- |
| // Accumulator second input. |
| val accvalid0 = io.op.dwinit || io.op.mul0.dmulh || io.op.mul0.mulh || io.op.add.add3 || io.op.mul0.madd || io.op.shf.shf |
| val accvalid1 = io.op.dwinit || io.op.mul1.dmulh || io.op.mul1.mulh |
| |
| val accum0 = MuxOR(io.op.add.add3 || |
| io.op.mul0.madd, io.read(2).data) | |
| MuxOR(io.op.mul0.dmulh || |
| io.op.mul0.mulh || |
| io.op.shf.shf, rnd0) | |
| MuxOR(io.op.dwinit, io.read(0).data) |
| |
| val accum1 = MuxOR(io.op.mul1.dmulh || |
| io.op.mul1.mulh, rnd1) | |
| MuxOR(io.op.dwinit, io.read(1).data) |
| |
| // --------------------------------------------------------------------------- |
| // Registration. |
| val wsz = RegInit(0.U(3.W)) |
| val waccvalid0 = RegInit(false.B) |
| val waccvalid1 = RegInit(false.B) |
| val wdata0 = Reg(UInt(32.W)) |
| val waccm0 = Reg(UInt(32.W)) |
| val wdata1 = Reg(UInt(32.W)) |
| val waccm1 = Reg(UInt(32.W)) |
| |
| wsz := MuxOR(io.in.vdvalid || io.in.vevalid, io.in.sz) |
| waccvalid0 := accvalid0 || io.op.dwconv |
| waccvalid1 := accvalid1 || io.op.dwconv |
| |
| when (io.in.vdvalid) { |
| wdata0 := out0 | io.load(0) |
| } |
| |
| when (accvalid0) { |
| waccm0 := accum0 |
| } .elsewhen (io.op.dwconvData) { |
| waccm0 := io.write(0).data |
| } |
| |
| when (io.in.vevalid) { |
| wdata1 := out1 | io.load(1) |
| } |
| |
| when (accvalid1) { |
| waccm1 := accum1 |
| } .elsewhen (io.op.dwconvData) { |
| waccm1 := io.write(1).data |
| } |
| |
| def Accum(en: Bool, d: UInt, a: UInt): UInt = { |
| val dm = MuxOR(en, d) |
| val am = MuxOR(en, a) |
| val rm = MuxOR(en && wsz(0), Cat(dm(31,24) + am(31,24), |
| dm(23,16) + am(23,16), |
| dm(15, 8) + am(15, 8), |
| dm( 7, 0) + am( 7, 0))) | |
| MuxOR(en && wsz(1), Cat(dm(31,16) + am(31,16), |
| dm(15, 0) + am(15, 0))) | |
| MuxOR(en && wsz(2), dm(31, 0) + am(31, 0)) |
| val rn = MuxOR(!en, d) |
| assert(rm.getWidth == 32) |
| assert(rn.getWidth == 32) |
| rm | rn |
| } |
| |
| io.write(0).data := Accum(waccvalid0, wdata0, waccm0) |
| io.write(1).data := Accum(waccvalid1, wdata1, waccm1) |
| } |
| |
| object EmitVAluInt extends App { |
| val p = new Parameters |
| ChiselStage.emitSystemVerilogFile(new VAluInt(p, 0), args) |
| } |
