Dell FPGA driver fix (#15144)

Why I did it
FPGA driver crash was observed in Dell FPGA based platforms.

How I did it
Fixed FPGA crash

How to verify it
Load FPGA driver and check whether the kernel crashes.

platform/broadcom/sonic-platform-modules-dell/common/dell_fpga_ocores.c

@@ -237,9 +237,6 @@ enum {
 #define MB_BRD_REV_TYPE             0x0008
 #define MB_BRD_REV_MASK             0x00f0
 #define MB_BRD_REV_00               0x0000
-#define MB_BRD_REV_01               0x0010
-#define MB_BRD_REV_02               0x0020
-#define MB_BRD_REV_03               0x0030
 #define MB_BRD_TYPE_MASK            0x000f
 #define BRD_TYPE_Z9232_NON_NEBS     0x0
 #define BRD_TYPE_Z9232_NEBS         0x1
@@ -256,6 +253,10 @@ enum {
 #define BRD_TYPE_S5232_NON_NEBS     0xc
 #define BRD_TYPE_S5232_NEBS         0xd
 
+#define SYS_SRR                     0x00F8
+#define SYS_SRR_MASK                0xFFFFFFFF
+#define SYS_SRR_RST_MAX             5
+
 #define FPGA_CTL_REG_SIZE           0x60
 #define MSI_VECTOR_MAP_MASK         0x1f
 #define MSI_VECTOR_MAP1             0x58
@@ -520,6 +521,11 @@ static void fpgai2c_process(struct fpgalogic_i2c *i2c)
     u8 stat = fpgai2c_reg_get(i2c, FPGAI2C_REG_STATUS);
 
     PRINT("fpgai2c_process in. status reg :0x%x\n", stat);
+    /* Clear spurious interrupts */
+    if (!(stat & FPGAI2C_REG_STAT_IF)) {
+        PRINT("spurious interrupt, status reg :0x%x\n", stat);
+        return;
+    }
 
     if ((i2c->state == STATE_STOP) || (i2c->state == STATE_ERROR)) {
         /* stop has been sent */
@@ -557,14 +563,23 @@ static void fpgai2c_process(struct fpgalogic_i2c *i2c)
             fpgai2c_stop(i2c);
             return;
         }
-    } else
+    } else if (i2c->state == STATE_READ) {
-    {
+        if (stat & FPGAI2C_REG_STAT_BUSY) {
             if ((i2c->msg == NULL) || (i2c->msg->buf == NULL) || (i2c->pos >= i2c->msg->len)) {
-            printk("crash debug..1 fpgai2c_process MSG and MAS->BUFF is NULL or pos > len ");
+                printk("DBG 1: fpgai2c_process I2C message or message buffer is NULL or \
+			pos is greater than I2C message length\n");
                 return;
             }
-
             msg->buf[i2c->pos++] = fpgai2c_reg_get(i2c, FPGAI2C_REG_DATA);
+        } else {
+            fpgai2c_reg_set(i2c, FPGAI2C_REG_CMD, FPGAI2C_REG_CMD_IACK);
+            PRINT("!FPGAI2C_REG_STAT_BUSY in STATE_READ clear interrupt\n");
+            return;
+        }
+    } else {
+        fpgai2c_reg_set(i2c, FPGAI2C_REG_CMD, FPGAI2C_REG_CMD_IACK);
+        PRINT("NOT START/WRITE/READ i2c->state 0x%x\n", i2c->state);
+        return;
     }
 
     /* end of msg? */
@@ -599,14 +614,21 @@ static void fpgai2c_process(struct fpgalogic_i2c *i2c)
         }
     }
 
-    if (i2c->state == STATE_READ) {
+    /* Add more protection */
-
     if ((i2c->msg == NULL) || (i2c->msg->buf == NULL) || (i2c->pos >= i2c->msg->len)) {
-            printk("crash debug..2 fpgai2c_process MSG and MAS->BUFF is NULL or pos > len ");
+        printk("DBG 2: fpgai2c_process I2C message or message buffer is NULL or \
+		pos is greater than I2C message length\n");
+        i2c->state = STATE_STOP;
+        fpgai2c_stop(i2c);
         return;
     }
 
-
+    if (i2c->state == STATE_READ) {
+	if ((i2c->msg == NULL) || (i2c->msg->buf == NULL) || (i2c->pos >= i2c->msg->len)) {
+		printk("DBG 3: fpgai2c_process I2C message or message buffer is NULL or \
+			pos is greater than I2C message length\n");
+		return;
+	}
         PRINT("fpgai2c_poll STATE_READ i2c->pos=%d msg->len-1 = 0x%x set FPGAI2C_REG_CMD = 0x%x\n",i2c->pos, msg->len-1,
                 i2c->pos == (msg->len-1) ?  FPGAI2C_REG_CMD_READ_NACK : FPGAI2C_REG_CMD_READ_ACK);
         fpgai2c_reg_set(i2c, FPGAI2C_REG_CMD, i2c->pos == (msg->len-1) ?
@@ -615,10 +637,10 @@ static void fpgai2c_process(struct fpgalogic_i2c *i2c)
         PRINT("fpgai2c_process set FPGAI2C_REG_DATA(0x%x)\n",FPGAI2C_REG_DATA);
 
 	if ((i2c->msg == NULL) || (i2c->msg->buf == NULL) || (i2c->pos >= i2c->msg->len)) {
-            printk("crash debug..3 fpgai2c_process MSG and MAS->BUFF is NULL or pos > len ");
+		printk("DBG 4: fpgai2c_process I2C message or message buffer is NULL or \
+			pos is greater than I2C message length\n");
 		return;
 	}
-
         fpgai2c_reg_set(i2c, FPGAI2C_REG_DATA, msg->buf[i2c->pos++]);
         fpgai2c_reg_set(i2c, FPGAI2C_REG_CMD, FPGAI2C_REG_CMD_WRITE);
     }
@@ -689,9 +711,8 @@ static int fpgai2c_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, int num)
                 ((i2c->msg->flags & I2C_M_RD) ? 1:0));
         fpgai2c_reg_set(i2c, FPGAI2C_REG_CMD, FPGAI2C_REG_CMD_START);
 
-        /* Interrupt mode */
+        /* Interrupt mode, wait for MSI, timeout in error conditions */
-        if (wait_event_timeout(i2c->wait, (i2c->state == STATE_ERROR) ||
+        if (wait_event_timeout(i2c->wait, (i2c->state == STATE_DONE), HZ/4))
-                       (i2c->state == STATE_DONE), HZ))
             ret = (i2c->state == STATE_DONE) ? num : -EIO;
         return ret;
     }
@@ -797,6 +818,18 @@ static int fpgai2c_interrupt_enable(struct fpgapci_dev *fpgapci)
     return 0;
 }
 
+static int fpgai2c_interrupt_disable(struct fpgapci_dev *fpgapci)
+{
+    int i;
+    u8 ctrl = 0;
+
+    /* disable Interrupts */
+    for (i = 0 ; i < total_i2c_pci_bus; i ++) {
+        fpgai2c_reg_set(&fpgalogic_i2c[i], FPGAI2C_REG_CONTROL, ctrl | FPGAI2C_REG_CTRL_EN);
+    }
+    return 0;
+}
+
 static u32 fpgai2c_func(struct i2c_adapter *adap)
 {
 	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
@@ -818,8 +851,8 @@ static int  i2c_pci_add_bus (struct i2c_adapter *adap)
 
 static int i2c_init_internal_data(void)
 {
-	int i;
+	int i, rst_cnt = 0;
-    PRINT("%s(), line:%d\n", __func__, __LINE__);
+	uint32_t sys_srr;
 
 	for( i = 0; i < total_i2c_pci_bus; i++ )
 	{
@@ -830,6 +863,22 @@ static int i2c_init_internal_data(void)
 		fpgalogic_i2c[i].bus_clock_khz = 100;
 		fpgalogic_i2c[i].base = fpga_base_addr + i*FPGALOGIC_CH_OFFSET;
 		mutex_init(&fpgalogic_i2c[i].lock);
+
+		/* Software reset each channel before initializing it */
+		iowrite32(~(0x1 << i) & SYS_SRR_MASK, fpga_ctl_addr + SYS_SRR);
+		do {
+			sys_srr = ioread32(fpga_ctl_addr + SYS_SRR);
+			if (sys_srr & (0x1 << i)) {
+				mdelay(40);
+			} else {
+				break;
+			}
+			rst_cnt++;
+		} while (rst_cnt < SYS_SRR_RST_MAX);
+
+		mdelay(1);
+		iowrite32(SYS_SRR_MASK, fpga_ctl_addr + SYS_SRR);
+
 		fpgai2c_init(&fpgalogic_i2c[i]);
 	}
 
@@ -1310,9 +1359,7 @@ static int fpgapci_configure_msi(struct fpgapci_dev *fpgapci,struct pci_dev *dev
             if (err < MSI_VECTOR_REV_00) {
                 goto error_disable_msi;
             }
-        } else if (((board_rev_type & MB_BRD_REV_MASK) == MB_BRD_REV_01) ||
+        } else {
-        ((board_rev_type & MB_BRD_REV_MASK) == MB_BRD_REV_02) ||
-        ((board_rev_type & MB_BRD_REV_MASK) == MB_BRD_REV_03)) {
             if (err < MSI_VECTOR_REV_01) {
                 goto error_disable_msi;
             }
@@ -1379,7 +1426,6 @@ static int fpgapci_probe(struct pci_dev *dev, const struct pci_device_id *id)
             }
             /* Enable interrupt after config msi */
             fpgai2c_interrupt_enable(fpgapci);
-        
         }
 
 
@@ -1415,8 +1461,12 @@ static void fpgapci_remove(struct pci_dev *dev)
 		PRINT ( ": fpgapci_dev is 0\n");
 		return;
 	}
+        
+        /* Disable interrupt before uninitialize device */
+        fpgai2c_interrupt_disable(fpgapci);
+
 	i2c_pci_deinit();
-	//
+
 	if (use_irq)
 	{
 		for(i = 0; i < fpgapci->irq_assigned; i++)

platform/broadcom/sonic-platform-modules-dell/debian/rules

@@ -31,31 +31,37 @@ override_dh_auto_build:
 			python3 setup.py bdist_wheel -d $(MOD_SRC_DIR)/$${mod}/modules; \
 			cd $(MOD_SRC_DIR); \
 		elif [ $$mod = "z9264f" ]; then \
+		        cp $(COMMON_DIR)/dell_fpga_ocores.c $(MOD_SRC_DIR)/$${mod}/modules/dell_z9264f_fpga_ocores.c; \
 			cp $(COMMON_DIR)/ipmihelper.py $(MOD_SRC_DIR)/$${mod}/sonic_platform/ipmihelper.py; \
 			cd $(MOD_SRC_DIR)/$${mod}; \
 			python3 setup.py bdist_wheel -d $(MOD_SRC_DIR)/$${mod}/modules; \
 			cd $(MOD_SRC_DIR); \
 		elif [ $$mod = "s5212f" ]; then \
+		        cp $(COMMON_DIR)/dell_fpga_ocores.c $(MOD_SRC_DIR)/$${mod}/modules/dell_s5212f_fpga_ocores.c; \
 			cp $(COMMON_DIR)/ipmihelper.py $(MOD_SRC_DIR)/$${mod}/sonic_platform/ipmihelper.py; \
 			cd $(MOD_SRC_DIR)/$${mod}; \
 			python3 setup.py bdist_wheel -d $(MOD_SRC_DIR)/$${mod}/modules; \
 			cd $(MOD_SRC_DIR); \
 		elif [ $$mod = "s5224f" ]; then \
+		        cp $(COMMON_DIR)/dell_fpga_ocores.c $(MOD_SRC_DIR)/$${mod}/modules/dell_s5224f_fpga_ocores.c; \
 			cp $(COMMON_DIR)/ipmihelper.py $(MOD_SRC_DIR)/$${mod}/sonic_platform/ipmihelper.py; \
 			cd $(MOD_SRC_DIR)/$${mod}; \
 			python3 setup.py bdist_wheel -d $(MOD_SRC_DIR)/$${mod}/modules; \
 			cd $(MOD_SRC_DIR); \
 		elif [ $$mod = "s5232f" ]; then \
+		        cp $(COMMON_DIR)/dell_fpga_ocores.c $(MOD_SRC_DIR)/$${mod}/modules/dell_s5232f_fpga_ocores.c; \
 			cp $(COMMON_DIR)/ipmihelper.py $(MOD_SRC_DIR)/$${mod}/sonic_platform/ipmihelper.py; \
 			cd $(MOD_SRC_DIR)/$${mod}; \
 			python3 setup.py bdist_wheel -d $(MOD_SRC_DIR)/$${mod}/modules; \
 			cd $(MOD_SRC_DIR); \
 		elif [ $$mod = "s5248f" ]; then \
+		        cp $(COMMON_DIR)/dell_fpga_ocores.c $(MOD_SRC_DIR)/$${mod}/modules/dell_s5248f_fpga_ocores.c; \
 			cp $(COMMON_DIR)/ipmihelper.py $(MOD_SRC_DIR)/$${mod}/sonic_platform/ipmihelper.py; \
 			cd $(MOD_SRC_DIR)/$${mod}; \
 			python3 setup.py bdist_wheel -d $(MOD_SRC_DIR)/$${mod}/modules; \
 			cd $(MOD_SRC_DIR); \
 		elif [ $$mod = "s5296f" ]; then \
+		        cp $(COMMON_DIR)/dell_fpga_ocores.c $(MOD_SRC_DIR)/$${mod}/modules/dell_s5296f_fpga_ocores.c; \
 			cp $(COMMON_DIR)/ipmihelper.py $(MOD_SRC_DIR)/$${mod}/sonic_platform/ipmihelper.py; \
 			cd $(MOD_SRC_DIR)/$${mod}; \
 			python3 setup.py bdist_wheel -d $(MOD_SRC_DIR)/$${mod}/modules; \
@@ -70,6 +76,7 @@ override_dh_auto_build:
 			python3 setup.py bdist_wheel -d $(MOD_SRC_DIR)/$${mod}/modules; \
 			cd $(MOD_SRC_DIR); \
 		elif [ $$mod = "z9432f" ]; then \
+		        cp $(COMMON_DIR)/dell_fpga_ocores.c $(MOD_SRC_DIR)/$${mod}/modules/dell_z9432f_fpga_ocores.c; \
 			cp $(COMMON_DIR)/ipmihelper.py $(MOD_SRC_DIR)/$${mod}/sonic_platform/ipmihelper.py; \
 			cd $(MOD_SRC_DIR)/$${mod}; \
 			python3 setup.py bdist_wheel -d $(MOD_SRC_DIR)/$${mod}/modules; \
@@ -124,31 +131,37 @@ override_dh_clean:
 			rm -rf $(MOD_SRC_DIR)/$${mod}/build; \
 			rm -rf $(MOD_SRC_DIR)/$${mod}/build/*.egg-info; \
 		elif [ $$mod = "z9264f" ]; then \
+		        rm -f $(MOD_SRC_DIR)/$${mod}/modules/dell_z9264f_fpga_ocores.c; \
 			rm -f $(MOD_SRC_DIR)/$${mod}/sonic_platform/ipmihelper.py; \
 			rm -f $(MOD_SRC_DIR)/$${mod}/modules/*.whl; \
 			rm -rf $(MOD_SRC_DIR)/$${mod}/build; \
 			rm -rf $(MOD_SRC_DIR)/$${mod}/build/*.egg-info; \
 		elif [ $$mod = "s5212f" ]; then \
     	rm -f $(MOD_SRC_DIR)/$${mod}/sonic_platform/ipmihelper.py; \
+	                rm -f $(MOD_SRC_DIR)/$${mod}/modules/dell_s5212f_fpga_ocores.c; \
 			rm -f $(MOD_SRC_DIR)/$${mod}/modules/*.whl; \
 			rm -rf $(MOD_SRC_DIR)/$${mod}/build; \
 			rm -rf $(MOD_SRC_DIR)/$${mod}/build/*.egg-info; \
 		elif [ $$mod = "s5224f" ]; then \
+		        rm -f $(MOD_SRC_DIR)/$${mod}/modules/dell_s5224f_fpga_ocores.c; \
 			rm -f $(MOD_SRC_DIR)/$${mod}/sonic_platform/ipmihelper.py; \
 			rm -f $(MOD_SRC_DIR)/$${mod}/modules/*.whl; \
 			rm -rf $(MOD_SRC_DIR)/$${mod}/build; \
 			rm -rf $(MOD_SRC_DIR)/$${mod}/build/*.egg-info; \
 		elif [ $$mod = "s5232f" ]; then \
+		        rm -f $(MOD_SRC_DIR)/$${mod}/modules/dell_s5232f_fpga_ocores.c; \
 			rm -f $(MOD_SRC_DIR)/$${mod}/sonic_platform/ipmihelper.py; \
 			rm -f $(MOD_SRC_DIR)/$${mod}/modules/*.whl; \
 			rm -rf $(MOD_SRC_DIR)/$${mod}/build; \
 			rm -rf $(MOD_SRC_DIR)/$${mod}/build/*.egg-info; \
 		elif [ $$mod = "s5248f" ]; then \
+		        rm -f $(MOD_SRC_DIR)/$${mod}/modules/dell_s5248f_fpga_ocores.c; \
 			rm -f $(MOD_SRC_DIR)/$${mod}/sonic_platform/ipmihelper.py; \
 			rm -f $(MOD_SRC_DIR)/$${mod}/modules/*.whl; \
 			rm -rf $(MOD_SRC_DIR)/$${mod}/build; \
 			rm -rf $(MOD_SRC_DIR)/$${mod}/build/*.egg-info; \
 		elif [ $$mod = "s5296f" ]; then \
+		        rm -f $(MOD_SRC_DIR)/$${mod}/modules/dell_s5296f_fpga_ocores.c; \
 			rm -f $(MOD_SRC_DIR)/$${mod}/sonic_platform/ipmihelper.py; \
 			rm -f $(MOD_SRC_DIR)/$${mod}/modules/*.whl; \
 			rm -rf $(MOD_SRC_DIR)/$${mod}/build; \
@@ -159,6 +172,7 @@ override_dh_clean:
 			rm -rf $(MOD_SRC_DIR)/$${mod}/build; \
 			rm -rf $(MOD_SRC_DIR)/$${mod}/build/*.egg-info; \
 		elif [ $$mod = "z9432f" ]; then \
+		        rm -f $(MOD_SRC_DIR)/$${mod}/modules/dell_z9432f_fpga_ocores.c; \
 			rm -f $(MOD_SRC_DIR)/$${mod}/sonic_platform/ipmihelper.py; \
 			rm -f $(MOD_SRC_DIR)/$${mod}/modules/*.whl; \
 			rm -rf $(MOD_SRC_DIR)/$${mod}/build; \