[dv/otp_ctrl] Support lc_escalate_en

This PR adds a sequence to support lc_esc_en.
This PR has basic sequence and scb support.

The following TODOs will be implemented in coming PRs:
1. Add assertions when lc_esc_en is On.
2. Add random input to lc_esc_en, not just On and Off
3. Support key request and lc_esc_en happened in parallel

Signed-off-by: Cindy Chen <chencindy@google.com>
diff --git a/hw/ip/otp_ctrl/dv/env/otp_ctrl_if.sv b/hw/ip/otp_ctrl/dv/env/otp_ctrl_if.sv
index b8778e0..2a14c25 100644
--- a/hw/ip/otp_ctrl/dv/env/otp_ctrl_if.sv
+++ b/hw/ip/otp_ctrl/dv/env/otp_ctrl_if.sv
@@ -24,17 +24,30 @@
   logic                lc_prog_req, lc_prog_err;
   logic                lc_prog_err_dly1, lc_prog_no_sta_check;
 
+  // Signals to skip csr check during first two clock cycles after lc_escalate_en is set.
+  // Because lc_escalate_en might take one clock cycle to propogate to design.
+  logic                lc_esc_dly1, lc_esc_dly2;
+  bit                  skip_read_check;
+
   // Lc_err could trigger during LC program, so check intr and status after lc_req is finished.
   // Lc_err takes one clock cycle to propogate to intr signal. So avoid intr check if it happens
   // during the transition.
   always_ff @(posedge clk_i or negedge rst_ni) begin
     if (!rst_ni) begin
       lc_prog_err_dly1 <= 0;
+      lc_esc_dly1      <= 0;
+      lc_esc_dly2      <= 0;
     end else begin
       lc_prog_err_dly1 <= lc_prog_err;
+      lc_esc_dly1      <= lc_escalate_en_i;
+      lc_esc_dly2      <= lc_esc_dly1;
     end
   end
-  assign lc_prog_no_sta_check = lc_prog_err | lc_prog_err_dly1 | lc_prog_req;
+
+  assign skip_read_check = (lc_escalate_en_i == lc_ctrl_pkg::On) &&
+                           (lc_esc_dly1 != lc_ctrl_pkg::On || lc_esc_dly2 != lc_ctrl_pkg::On);
+  assign lc_prog_no_sta_check = lc_prog_err | lc_prog_err_dly1 | lc_prog_req |
+                                lc_escalate_en_i == lc_ctrl_pkg::On;
 
   // TODO: for lc_tx, except esc_en signal, all value different from On is treated as Off,
   // technically we can randomize values here once scb supports
@@ -55,20 +68,33 @@
     lc_dft_en_i = val;
   endtask
 
+  task automatic drive_lc_escalate_en(lc_ctrl_pkg::lc_tx_e val);
+    lc_escalate_en_i = val;
+  endtask
+
+  `define OTP_ASSERT_WO_LC_ESC(NAME, SEQ) \
+    `ASSERT(NAME, SEQ, clk_i, !rst_ni || lc_escalate_en_i == lc_ctrl_pkg::On)
+
   // If pwr_otp_idle is set only if pwr_otp init is done
-  `ASSERT(OtpPwrDoneWhenIdle_A, pwr_otp_idle_o |-> pwr_otp_done_o)
+  `OTP_ASSERT_WO_LC_ESC(OtpPwrDoneWhenIdle_A, pwr_otp_idle_o |-> pwr_otp_done_o)
 
   // Otp_hw_cfg_o is valid only when otp init is done
-  `ASSERT(OtpHwCfgValidOn_A, pwr_otp_done_o |-> otp_hw_cfg_o.valid == lc_ctrl_pkg::On)
+  `OTP_ASSERT_WO_LC_ESC(OtpHwCfgValidOn_A, pwr_otp_done_o |->
+                        otp_hw_cfg_o.valid == lc_ctrl_pkg::On)
   // If otp_hw_cfg is Off, then hw partition is not finished calculation, then otp init is not done
-  `ASSERT(OtpHwCfgValidOff_A, otp_hw_cfg_o.valid == lc_ctrl_pkg::Off |-> pwr_otp_done_o == 0)
+  `OTP_ASSERT_WO_LC_ESC(OtpHwCfgValidOff_A, otp_hw_cfg_o.valid == lc_ctrl_pkg::Off |->
+                        pwr_otp_done_o == 0)
   // Once OTP init is done, hw_cfg_o output value stays stable until next power cycle
-  `ASSERT(OtpHwCfgStable_A, otp_hw_cfg_o.valid == lc_ctrl_pkg::On |=> $stable(otp_hw_cfg_o))
+  `OTP_ASSERT_WO_LC_ESC(OtpHwCfgStable_A, otp_hw_cfg_o.valid == lc_ctrl_pkg::On |=>
+                        $stable(otp_hw_cfg_o))
 
   // Otp_keymgr valid is related to part_digest, should not be changed after otp_pwr_init
-  `ASSERT(OtpKeymgrValidStable_A, pwr_otp_done_o |-> $stable(keymgr_key_o.valid))
+  `OTP_ASSERT_WO_LC_ESC(OtpKeymgrValidStable_A, pwr_otp_done_o |-> $stable(keymgr_key_o.valid))
 
   // During lc_prog_req, either otp_idle will be reset or lc_error is set
-  `ASSERT(LcProgReq_A, $rose(lc_prog_req) |=>
+  `OTP_ASSERT_WO_LC_ESC(LcProgReq_A, $rose(lc_prog_req) |=>
                        (pwr_otp_idle_o == 0 || $rose(lc_prog_err)) within lc_prog_req[*1:$])
+
+  // TODO: add assertions when esc_en is On
+  `undef OTP_ASSERT_WO_LC_ESC
 endinterface
diff --git a/hw/ip/otp_ctrl/dv/env/otp_ctrl_scoreboard.sv b/hw/ip/otp_ctrl/dv/env/otp_ctrl_scoreboard.sv
index dc0ac6a..8f1b1e7 100644
--- a/hw/ip/otp_ctrl/dv/env/otp_ctrl_scoreboard.sv
+++ b/hw/ip/otp_ctrl/dv/env/otp_ctrl_scoreboard.sv
@@ -32,6 +32,8 @@
 
   bit macro_alert_triggered;
 
+  bit lc_esc;
+
   // TLM agent fifos
   uvm_tlm_analysis_fifo #(push_pull_item#(.DeviceDataWidth(SRAM_DATA_SIZE)))
                         sram_fifos[NumSramKeyReqSlots];
@@ -67,6 +69,7 @@
     super.run_phase(phase);
     fork
       process_otp_power_up();
+      process_lc_esc();
       process_lc_token_req();
       process_lc_prog_req();
       process_edn_req();
@@ -116,13 +119,15 @@
           bit [otp_ctrl_pkg::KeyMgrKeyWidth-1:0] exp_keymgr_key0, exp_keymgr_key1;
 
           // Dai access is unlocked because the power init is done
-          void'(ral.direct_access_regwen.predict(1));
+          if (!lc_esc) void'(ral.direct_access_regwen.predict(1));
 
           // Dai idle is set because the otp init is done
           exp_status[OtpDaiIdleIdx] = 1;
 
           // Hwcfg_o gets data from OTP HW cfg partition
-          exp_hwcfg_data = otp_hw_cfg_data_t'({<<32 {otp_a[HwCfgOffset/4 +: HwCfgSize/4]}});
+          exp_hwcfg_data = lc_esc ?
+                           otp_ctrl_part_pkg::PartInvDefault[HwCfgOffset*8 +: HwCfgSize*8] :
+                           otp_hw_cfg_data_t'({<<32 {otp_a[HwCfgOffset/4 +: HwCfgSize/4]}});
           `DV_CHECK_EQ(cfg.otp_ctrl_vif.otp_hw_cfg_o.data, exp_hwcfg_data)
 
           // Otp_keymgr outputs creator root key shares from the secret2 partition.
@@ -140,6 +145,30 @@
     end
   endtask
 
+  virtual task process_lc_esc();
+    forever begin
+      wait(cfg.otp_ctrl_vif.lc_escalate_en_i == lc_ctrl_pkg::On);
+      // LC_escalate_en will trigger fatal check alert.
+      set_exp_alert("fatal_check_error", 1, 5);
+
+      // Update status bits.
+      exp_status = '0;
+      for (int i = 0; i < OtpTimeoutErrIdx; i++) predict_err(otp_status_e'(i), OtpFsmStateError);
+      exp_status[OtpDerivKeyFsmErrIdx:OtpLfsrFsmErrIdx] = '1;
+
+      // Set lc_esc flag, only DUT reset can clear this flag.
+      lc_esc = 1;
+
+      // Update digest values and direct_access_regwen.
+      for (int i = HwCfgIdx; i <= Secret2Idx; i++) digests[i] = 0;
+      predict_digest_csrs();
+      predict_rdata(1, 0, 0);
+      void'(ral.direct_access_regwen.predict(.value(0), .kind(UVM_PREDICT_READ)));
+
+      wait(cfg.otp_ctrl_vif.lc_escalate_en_i != lc_ctrl_pkg::On);
+    end
+  endtask
+
   virtual task process_lc_prog_req();
     forever begin
       push_pull_item#(.DeviceDataWidth(1), .HostDataWidth(LC_PROG_DATA_SIZE)) rcv_item;
@@ -349,7 +378,7 @@
   virtual task process_tl_access(tl_seq_item item, tl_channels_e channel = DataChannel);
     uvm_reg     csr;
     dv_base_reg dv_reg;
-    bit         do_read_check = 1'b1;
+    bit         do_read_check = !cfg.otp_ctrl_vif.skip_read_check;
     bit         write         = item.is_write();
     uvm_reg_addr_t csr_addr   = ral.get_word_aligned_addr(item.a_addr);
     bit [TL_AW-1:0] addr_mask = ral.get_addr_mask();
@@ -432,7 +461,7 @@
         end
       end
       "direct_access_cmd": begin
-        if (addr_phase_write) begin
+        if (addr_phase_write && !lc_esc) begin
           // here only normalize to 2 lsb, if is secret, will be reduced further
           bit [TL_AW-1:0] dai_addr = `gmv(ral.direct_access_address) >> 2 << 2;
           int part_idx = get_part_index(dai_addr);
@@ -568,8 +597,10 @@
           if (item.d_data[OtpDaiIdleIdx]) check_otp_idle(1);
 
           // STATUS register check with mask
-          `DV_CHECK_EQ((csr.get_mirrored_value() | status_mask), (item.d_data | status_mask),
-                       $sformatf("reg name: status, compare_mask %0h", status_mask))
+          if (do_read_check) begin
+            `DV_CHECK_EQ((csr.get_mirrored_value() | status_mask), (item.d_data | status_mask),
+                         $sformatf("reg name: status, compare_mask %0h", status_mask))
+          end
 
           // Check if OtpCheckPending is set correctly, then ignore checking until check is done
           if (under_chk) begin
@@ -579,7 +610,7 @@
             end
           end
 
-          if (under_dai_access) begin
+          if (under_dai_access && !lc_esc) begin
             if (item.d_data[OtpDaiIdleIdx]) begin
               under_dai_access = 0;
               void'(ral.direct_access_regwen.predict(1));
@@ -636,6 +667,7 @@
       sram_fifos[i].flush();
     end
 
+    lc_esc                = 0;
     under_chk             = 0;
     under_dai_access      = 0;
     macro_alert_triggered = 0;
@@ -671,7 +703,8 @@
                  cfg.m_otbn_pull_agent_cfg.vif.req || cfg.m_flash_data_pull_agent_cfg.vif.req ||
                  cfg.m_flash_addr_pull_agent_cfg.vif.req ||
                  cfg.m_sram_pull_agent_cfg[0].vif.req || cfg.m_sram_pull_agent_cfg[1].vif.req ||
-                 cfg.m_lc_prog_pull_agent_cfg.vif.req || cfg.m_lc_token_pull_agent_cfg.vif.req);
+                 cfg.m_lc_prog_pull_agent_cfg.vif.req || cfg.m_lc_token_pull_agent_cfg.vif.req ||
+                 lc_esc);
           end
         join_any
         disable fork;
@@ -748,6 +781,8 @@
     int             array_size;
     real            key_factor  = SCRAMBLE_KEY_SIZE / TL_DW;
 
+    if (lc_esc) return;
+
     if (digests[part_idx] != 0 ||
         part_idx inside {CreatorSwCfgIdx, OwnerSwCfgIdx, LifeCycleIdx}) begin
       predict_err(OtpDaiErrIdx, OtpAccessError);
@@ -870,6 +905,8 @@
   // This function predict OTP error related registers: intr_state, status, and err_code
   virtual function void predict_err(otp_status_e   status_err_idx,
                                     otp_err_code_e err_code = OtpNoError);
+    if (lc_esc) return;
+
     // Update intr_state
     void'(ral.intr_state.otp_error.predict(.value(1), .kind(UVM_PREDICT_READ)));
 
@@ -890,6 +927,8 @@
 
   // TODO: consider combine it with function predict_err()
   virtual function void predict_no_err(otp_status_e status_err_idx);
+    if (lc_esc) return;
+
     exp_status[status_err_idx] = 0;
     if (status_err_idx == OtpDaiErrIdx) exp_status[OtpDaiIdleIdx] = 1;
 
@@ -902,8 +941,10 @@
 
   virtual function void predict_rdata(bit is_64_bits, bit [TL_DW-1:0] rdata0,
                                       bit [TL_DW-1:0] rdata1 = 0);
-    void'(ral.direct_access_rdata_0.predict(rdata0));
-    if (is_64_bits) void'(ral.direct_access_rdata_1.predict(rdata1));
+    void'(ral.direct_access_rdata_0.predict(.value(rdata0), .kind(UVM_PREDICT_READ)));
+    if (is_64_bits) begin
+      void'(ral.direct_access_rdata_1.predict(.value(rdata1), .kind(UVM_PREDICT_READ)));
+    end
   endfunction
 
   // this function retrieves keys (128 bits) from scb's otp_array with a starting address
diff --git a/hw/ip/otp_ctrl/dv/env/seq_lib/otp_ctrl_base_vseq.sv b/hw/ip/otp_ctrl/dv/env/seq_lib/otp_ctrl_base_vseq.sv
index 91d6bcd..275a76c 100644
--- a/hw/ip/otp_ctrl/dv/env/seq_lib/otp_ctrl_base_vseq.sv
+++ b/hw/ip/otp_ctrl/dv/env/seq_lib/otp_ctrl_base_vseq.sv
@@ -81,12 +81,12 @@
     //   detected
     // - zero delays in TLUL interface, otherwise dai operation might be finished before reading
     //   these two CSRs
-    if (cfg.zero_delays && is_valid_dai_op) begin
+    if (cfg.zero_delays && is_valid_dai_op &&
+        cfg.otp_ctrl_vif.lc_escalate_en_i != lc_ctrl_pkg::On) begin
       csr_rd_check(ral.status.dai_idle, .compare_value(0), .backdoor(1));
       if ($urandom_range(0, 1)) csr_rd(.ptr(ral.direct_access_regwen), .value(val));
     end
-    if (cfg.ecc_err == OtpEccUncorrErr) csr_spinwait(ral.status.dai_error, 1);
-    else                                csr_spinwait(ral.status.dai_idle, 1);
+    wait_dai_op_done();
     rd_and_clear_intrs();
   endtask : dai_wr
 
@@ -107,14 +107,13 @@
     csr_wr(ral.direct_access_address, addr);
     csr_wr(ral.direct_access_cmd, int'(otp_ctrl_pkg::DaiRead));
 
-    if (cfg.zero_delays && is_valid_dai_op) begin
+    if (cfg.zero_delays && is_valid_dai_op &&
+        cfg.otp_ctrl_vif.lc_escalate_en_i != lc_ctrl_pkg::On) begin
       csr_rd_check(ral.status.dai_idle, .compare_value(0), .backdoor(1));
       if ($urandom_range(0, 1)) csr_rd(.ptr(ral.direct_access_regwen), .value(val));
     end
 
-    if (cfg.ecc_err == OtpEccUncorrErr) csr_spinwait(ral.status.dai_error, 1);
-    else                                csr_spinwait(ral.status.dai_idle, 1);
-
+    wait_dai_op_done();
     csr_rd(ral.direct_access_rdata_0, rdata0);
     if (is_secret(addr) || is_digest(addr)) csr_rd(ral.direct_access_rdata_1, rdata1);
     rd_and_clear_intrs();
@@ -143,13 +142,13 @@
     csr_wr(ral.direct_access_address, PART_BASE_ADDRS[part_idx]);
     csr_wr(ral.direct_access_cmd, otp_ctrl_pkg::DaiDigest);
 
-    if (cfg.zero_delays && is_valid_dai_op) begin
+    if (cfg.zero_delays && is_valid_dai_op &&
+        cfg.otp_ctrl_vif.lc_escalate_en_i != lc_ctrl_pkg::On) begin
       csr_rd_check(ral.status.dai_idle, .compare_value(0), .backdoor(1));
       if ($urandom_range(0, 1)) csr_rd(.ptr(ral.direct_access_regwen), .value(val));
     end
 
-    if (cfg.ecc_err == OtpEccUncorrErr) csr_spinwait(ral.status.dai_error, 1);
-    else                                csr_spinwait(ral.status.dai_idle, 1);
+    wait_dai_op_done();
     rd_and_clear_intrs();
   endtask
 
@@ -236,6 +235,31 @@
     if (wait_done && val) csr_spinwait(ral.status.check_pending, 0);
   endtask
 
+  // For a DAI interface operation to finish, either way until status dai_idle is set, or check
+  // err_code and see if fatal error happened.
+  virtual task wait_dai_op_done();
+    fork begin
+      fork
+        begin
+          csr_spinwait(.ptr(ral.status.dai_idle),
+                       .exp_data(1),
+                       .spinwait_delay_ns($urandom_range(0, 5)));
+        end
+        begin
+          forever begin
+            bit [TL_DW-1:0] err_val;
+            cfg.clk_rst_vif.wait_clks(1);
+            csr_rd(.ptr(ral.err_code.err_code_7), .value(err_val), .backdoor(1));
+            // Break if error will cause fatal alerts
+            if (err_val inside {OtpMacroEccUncorrError, OtpFsmStateError}) break;
+          end
+        end
+      join_any
+      wait_no_outstanding_access();
+      disable fork;
+    end join
+  endtask
+
   virtual task rd_and_clear_intrs();
     bit [TL_DW-1:0] val;
     if (cfg.otp_ctrl_vif.lc_prog_no_sta_check == 0) begin
@@ -293,7 +317,8 @@
     end
 
     `DV_CHECK_RANDOMIZE_FATAL(lc_prog_pull_seq)
-    `uvm_send(lc_prog_pull_seq)
+    `DV_SPINWAIT_EXIT(`uvm_send(lc_prog_pull_seq),
+                      wait(cfg.otp_ctrl_vif.lc_escalate_en_i == lc_ctrl_pkg::On);)
 
     if (check_intr) rd_and_clear_intrs();
   endtask
@@ -302,7 +327,8 @@
     push_pull_host_seq#(.HostDataWidth(lc_ctrl_state_pkg::LcTokenWidth)) lc_token_pull_seq;
     `uvm_create_on(lc_token_pull_seq, p_sequencer.lc_token_pull_sequencer_h);
     `DV_CHECK_RANDOMIZE_FATAL(lc_token_pull_seq)
-    `uvm_send(lc_token_pull_seq)
+    `DV_SPINWAIT_EXIT(`uvm_send(lc_token_pull_seq),
+                      wait(cfg.otp_ctrl_vif.lc_escalate_en_i == lc_ctrl_pkg::On);)
   endtask
 
   // first two or three LSB bits of DAI address can be randomized based on if it is secret
diff --git a/hw/ip/otp_ctrl/dv/env/seq_lib/otp_ctrl_parallel_lc_req_vseq.sv b/hw/ip/otp_ctrl/dv/env/seq_lib/otp_ctrl_parallel_lc_req_vseq.sv
index 8230aa0..31e7d7c 100644
--- a/hw/ip/otp_ctrl/dv/env/seq_lib/otp_ctrl_parallel_lc_req_vseq.sv
+++ b/hw/ip/otp_ctrl/dv/env/seq_lib/otp_ctrl_parallel_lc_req_vseq.sv
@@ -17,6 +17,8 @@
 
   constraint lc_trans_c {
     do_lc_trans == 0;
+    // TODO: support enable req_key while lc_escalate_en is set.
+    do_req_keys == 0;
   }
 
   // disable err_code check because cannot accurately predict when LC error is detected
@@ -31,7 +33,6 @@
 
         fork
           begin
-            // request lc transition
             if ($urandom_range(0, 1)) begin
               wait_clk_or_reset($urandom_range(0, 500));
               if (!base_vseq_done && !cfg.under_reset) req_lc_transition();
@@ -44,14 +45,31 @@
               if (!base_vseq_done && !cfg.under_reset) req_lc_token();
             end
           end
+          begin
+            // req lc token request
+            if ($urandom_range(0, 1)) begin
+              wait_clk_or_reset($urandom_range(0, 500));
+              if (!base_vseq_done && !cfg.under_reset) begin
+                // TODO: random drive any values instead of just on and off
+                cfg.otp_ctrl_vif.drive_lc_escalate_en(lc_ctrl_pkg::On);
+                // TODO: check with designer if we can take away this asssertoff
+                $assertoff(0, "tb.dut.u_otp_arb");
+                // Turn off reset because if issuing lc_escalation_en during otp program, scb cannot
+                // predict if the OTP memory is programmed or not.
+                do_reset_in_seq = 0;
+              end
+            end
+          end
         join
       end
   endtask
 
   // Use reset to clear lc interrupt error
   virtual task post_start();
-    if (do_apply_reset) apply_reset();
-    else wait(0); // wait until upper seq resets and kills this seq
+    if (do_apply_reset) begin
+      apply_reset();
+      cfg.otp_ctrl_vif.drive_lc_escalate_en(lc_ctrl_pkg::Off);
+    end else wait(0); // wait until upper seq resets and kills this seq
 
     // delay to avoid race condition when sending item and checking no item after reset occur
     // at the same time
diff --git a/hw/ip/otp_ctrl/dv/env/seq_lib/otp_ctrl_smoke_vseq.sv b/hw/ip/otp_ctrl/dv/env/seq_lib/otp_ctrl_smoke_vseq.sv
index a636b6a..a0ec771 100644
--- a/hw/ip/otp_ctrl/dv/env/seq_lib/otp_ctrl_smoke_vseq.sv
+++ b/hw/ip/otp_ctrl/dv/env/seq_lib/otp_ctrl_smoke_vseq.sv
@@ -12,6 +12,7 @@
   `uvm_object_new
 
   bit collect_used_addr = 1;
+  bit do_reset_in_seq = 1;
 
   rand bit                           do_req_keys, do_lc_trans, check_err_code;
   rand bit                           access_locked_parts;
@@ -62,6 +63,7 @@
   constraint ecc_err_c {ecc_err_mask == 0;}
 
   virtual task dut_init(string reset_kind = "HARD");
+    if (!do_reset_in_seq) return;
     super.dut_init(reset_kind);
     csr_wr(ral.intr_enable, en_intr);
   endtask
@@ -82,7 +84,9 @@
         if (i > 1) dut_init();
         // after otp-init done, check status
         cfg.clk_rst_vif.wait_clks(1);
-        csr_rd_check(.ptr(ral.status.dai_idle), .compare_value(1));
+        if ( cfg.otp_ctrl_vif.lc_escalate_en_i != lc_ctrl_pkg::On) begin
+          csr_rd_check(.ptr(ral.status.dai_idle), .compare_value(1));
+        end
       end
       do_otp_ctrl_init = 0;