[devices]: DellEmc-Z9264f:Interrupt based Optic OIR (#2926)

device/dell/x86_64-dellemc_z9264f_c3538-r0/plugins/sfputil.py

@@ -8,6 +8,7 @@ try:
     import sys
     import getopt
     import time
+    import select
     from sonic_sfp.sfputilbase import SfpUtilBase
     from os import *
     from mmap import *
@@ -24,6 +25,10 @@ class SfpUtil(SfpUtilBase):
     PORTS_IN_BLOCK = 64
 
     BASE_RES_PATH = "/sys/bus/pci/devices/0000:04:00.0/resource0"
+    OIR_FD_PATH = "/sys/bus/pci/devices/0000:04:00.0/port_msi"
+
+    oir_fd = -1
+    epoll = -1
 
     _port_to_eeprom_mapping = {}
 
@@ -47,8 +52,8 @@ class SfpUtil(SfpUtilBase):
 
     def pci_mem_read(self, mm, offset):
         mm.seek(offset)
-        read_data_stream=mm.read(4)
+        read_data_stream = mm.read(4)
-        reg_val=struct.unpack('I',read_data_stream)
+        reg_val = struct.unpack('I', read_data_stream)
         mem_val = str(reg_val)[1:-2]
         # print "reg_val read:%x"%reg_val
         return mem_val
@@ -56,7 +61,7 @@ class SfpUtil(SfpUtilBase):
     def pci_mem_write(self, mm, offset, data):
         mm.seek(offset)
         # print "data to write:%x"%data
-        mm.write(struct.pack('I',data))
+        mm.write(struct.pack('I', data))
 
     def pci_set_value(self, resource, val, offset):
         fd = open(resource, O_RDWR)
@@ -86,12 +91,10 @@ class SfpUtil(SfpUtilBase):
         eeprom_path = "/sys/class/i2c-adapter/i2c-{0}/{0}-0050/eeprom"
 
         for x in range(self.port_start, self.port_end + 1):
-            port_num  = x + 1
+            port_num = x + 1
-            self.port_to_eeprom_mapping[x] = eeprom_path.format(
+            self.port_to_eeprom_mapping[x] = eeprom_path.format(port_num)
-                        port_num)
             port_num = 0
         self.init_global_port_presence()
-        
         SfpUtilBase.__init__(self)
 
     def get_presence(self, port_num):
@@ -100,13 +103,13 @@ class SfpUtil(SfpUtilBase):
             return False
 
         # Port offset starts with 0x4004
-	port_offset = 16388 + ((port_num-1) * 16)
+        port_offset = 16388 + ((port_num-1) * 16)
 
-	status = self.pci_get_value(self.BASE_RES_PATH, port_offset)
+        status = self.pci_get_value(self.BASE_RES_PATH, port_offset)
-	reg_value = int(status)
+        reg_value = int(status)
 
         # Absence of status throws error
-        if (reg_value == "" ):
+        if (reg_value == ""):
             return False
 
         # Mask off 4th bit for presence
@@ -124,14 +127,14 @@ class SfpUtil(SfpUtilBase):
         if port_num < self.port_start or port_num > self.port_end:
             return False
 
-	# Port offset starts with 0x4000
+        # Port offset starts with 0x4000
-	port_offset = 16384 + ((port_num-1) * 16)
+        port_offset = 16384 + ((port_num-1) * 16)
 
-	status = self.pci_get_value(self.BASE_RES_PATH, port_offset)
+        status = self.pci_get_value(self.BASE_RES_PATH, port_offset)
-	reg_value = int(status)
+        reg_value = int(status)
 
         # Absence of status throws error
-        if (reg_value == "" ):
+        if (reg_value == ""):
             return False
 
         # Mask off 4th bit for presence
@@ -149,44 +152,44 @@ class SfpUtil(SfpUtilBase):
         if port_num < self.port_start or port_num > self.port_end:
             return False
 
-	# Port offset starts with 0x4000
+        # Port offset starts with 0x4000
-	port_offset = 16384 + ((port_num-1) * 16)
+        port_offset = 16384 + ((port_num-1) * 16)
 
-	status = self.pci_get_value(self.BASE_RES_PATH, port_offset)
+        status = self.pci_get_value(self.BASE_RES_PATH, port_offset)
-	reg_value = int(status)
+        reg_value = int(status)
 
         # Absence of status throws error
-        if (reg_value == "" ):
+        if (reg_value == ""):
             return False
 
         # Mask off 4th bit for presence
         mask = (1 << 6)
 
-	# LPMode is active high; set or clear the bit accordingly
+        # LPMode is active high; set or clear the bit accordingly
         if lpmode is True:
             reg_value = reg_value | mask
         else:
             reg_value = reg_value & ~mask
 
         # Convert our register value back to a hex string and write back
-	status = self.pci_set_value(self.BASE_RES_PATH, reg_value, port_offset)
+        status = self.pci_set_value(self.BASE_RES_PATH, reg_value, port_offset)
 
         return True
 
     def reset(self, port_num):
 
-	# Check for invalid port_num
+        # Check for invalid port_num
         if port_num < self.port_start or port_num > self.port_end:
             return False
 
-	# Port offset starts with 0x4000
+        # Port offset starts with 0x4000
-	port_offset = 16384 + ((port_num-1) * 16)
+        port_offset = 16384 + ((port_num-1) * 16)
 
-	status = self.pci_get_value(self.BASE_RES_PATH, port_offset)
+        status = self.pci_get_value(self.BASE_RES_PATH, port_offset)
-	reg_value = int(status)
+        reg_value = int(status)
 
         # Absence of status throws error
-        if (reg_value == "" ):
+        if (reg_value == ""):
             return False
 
         # Mask off 4th bit for presence
@@ -195,33 +198,106 @@ class SfpUtil(SfpUtilBase):
         # ResetL is active low
         reg_value = reg_value & ~mask
 
-	# Convert our register value back to a hex string and write back
+        # Convert our register value back to a hex string and write back
-	status = self.pci_set_value(self.BASE_RES_PATH, reg_value, port_offset)
+        status = self.pci_set_value(self.BASE_RES_PATH, reg_value, port_offset)
 
         # Sleep 1 second to allow it to settle
         time.sleep(1)
 
         reg_value = reg_value | mask
 
-	# Convert our register value back to a hex string and write back
+        # Convert our register value back to a hex string and write back
-	status = self.pci_set_value(self.BASE_RES_PATH, reg_value, port_offset)
+        status = self.pci_set_value(self.BASE_RES_PATH, reg_value, port_offset)
 
         return True
 
-    def get_transceiver_change_event(self):
+    def get_register(self, reg_file):
-        port_dict = {}
+            retval = 'ERR'
-        while True:
+            if (not path.isfile(reg_file)):
+                print reg_file,  'not found !'
+                return retval
+
+            try:
+                with fdopen(open(reg_file, O_RDONLY)) as fd:
+                    retval = fd.read()
+            except Exception as error:
+                logging.error("Unable to open ", reg_file, "file !")
+
+            retval = retval.rstrip('\r\n')
+            retval = retval.lstrip(" ")
+            return retval
+
+    def check_interrupts(self, port_dict):
+            retval = 0
+            is_port_dict_updated = False
             for port_num in range(self.port_start, (self.port_end + 1)):
                 presence = self.get_presence(port_num)
                 if(presence and self._global_port_pres_dict[port_num] == '0'):
+                    is_port_dict_updated = True
                     self._global_port_pres_dict[port_num] = '1'
                     port_dict[port_num] = '1'
                 elif(not presence and
                      self._global_port_pres_dict[port_num] == '1'):
+                    is_port_dict_updated = True
                     self._global_port_pres_dict[port_num] = '0'
                     port_dict[port_num] = '0'
+            return retval, is_port_dict_updated
 
-                if(len(port_dict) > 0):
+    def get_transceiver_change_event(self, timeout=0):
-                    return True, port_dict
+            port_dict = {}
+            try:
+                # We get notified when there is a MSI interrupt (vector 4/5)CVR
+                # Open the sysfs file and register the epoll object
+                self.oir_fd = fdopen(open(self.OIR_FD_PATH, O_RDONLY))
+                if self.oir_fd != -1:
+                    # Do a dummy read before epoll register
+                    self.oir_fd.read()
+                    self.epoll = select.epoll()
+                    self.epoll.register(
+                        self.oir_fd.fileno(), select.EPOLLIN & select.EPOLLET)
+                else:
+                    print("get_transceiver_change_event : unable to create fd")
+                    return False, {}
 
-            time.sleep(0.5)
+                # Check for missed interrupts by invoking self.check_interrupts
+                # which will update the port_dict.
+                while True:
+                    interrupt_count_start = self.get_register(self.OIR_FD_PATH)
+                    retval, is_port_dict_updated = \
+                        self.check_interrupts(port_dict)
+                    if ((retval == 0) and (is_port_dict_updated is True)):
+                        return True, port_dict
+                    interrupt_count_end = self.get_register(self.OIR_FD_PATH)
+                    if (interrupt_count_start == 'ERR' or
+                            interrupt_count_end == 'ERR'):
+                        print("get_transceiver_change_event : \
+                            unable to retrive interrupt count")
+                        break
+
+                    # check_interrupts() itself may take upto 100s of msecs.
+                    # We detect a missed interrupt based on the count
+                    if interrupt_count_start == interrupt_count_end:
+                        break
+
+                # Block until an xcvr is inserted or removed with timeout = -1
+                events = self.epoll.poll(
+                    timeout=timeout if timeout != 0 else -1)
+                if events:
+                    # check interrupts and return the port_dict
+                    retval, is_port_dict_updated = \
+                                              self.check_interrupts(port_dict)
+                    if (retval != 0):
+                        return False, {}
+
+                return True, port_dict
+            except:
+                return False, {}
+            finally:
+                if self.oir_fd != -1:
+                    self.epoll.unregister(self.oir_fd.fileno())
+                    self.epoll.close()
+                    self.oir_fd.close()
+                    self.oir_fd = -1
+                    self.epoll = -1
+
+            return False, {}

platform/broadcom/sonic-platform-modules-dell/debian/platform-modules-z9264f.install

@@ -3,6 +3,7 @@ z9264f/scripts/check_qsfp.sh usr/local/bin
 z9264f/scripts/platform_sensors.py usr/local/bin
 z9264f/scripts/sensors usr/bin
 z9264f/scripts/pcisysfs.py usr/bin
+z9264f/scripts/qsfp_irq_enable.py usr/bin
 z9264f/cfg/z9264f-modules.conf etc/modules-load.d
 z9264f/systemd/platform-modules-z9264f.service etc/systemd/system
 common/platform_reboot usr/share/sonic/device/x86_64-dellemc_z9264f_c3538-r0

platform/broadcom/sonic-platform-modules-dell/z9264f/modules/dell_z9264f_fpga_ocores.c

@@ -19,6 +19,7 @@
  * @file fpga_ocores.c
  * @brief This is a driver to interface with Linux Open Cores driver for FPGA i2c access
  *
+ * 2019/5/9 - PCIe Interrupt supported is added for OIR Events.
  ************************************************************************/
 #include <linux/kobject.h>
 #include <linux/kdev_t.h>
@@ -89,7 +90,7 @@ struct fpgapci_dev {
     unsigned int irq_first;
     unsigned int irq_length;
     unsigned int irq_assigned;
-
+    unsigned int xcvr_intr_count;
 };
 
 static int use_irq = 1;
@@ -226,6 +227,14 @@ enum {
 #define I2C_PCI_BUS_NUM_12          12
 #define I2C_PCI_BUS_NUM_16          16
 
+
+#define IRQ_LTCH_STS        		0x20
+#define PRSNT_LTCH_STS	        	0x10
+
+#define PORT_CTRL_OFFSET            0x4000
+#define PORT_STS_OFFSET             0x4004
+#define PORT_IRQ_STS_OFFSET         0x4008
+#define PORT_IRQ_EN_OFFSET          0x400C
 #define MB_BRD_REV_TYPE             0x0008
 #define MB_BRD_REV_MASK             0x00f0
 #define MB_BRD_REV_00               0x0000
@@ -248,7 +257,7 @@ enum {
 #define BRD_TYPE_S5232_NON_NEBS     0xc
 #define BRD_TYPE_S5232_NEBS         0xd
 
-#define FPGA_CTL_REG_SIZE           0x60
+#define FPGA_CTL_REG_SIZE           0x6000
 #define MSI_VECTOR_MAP_MASK         0x1f
 #define MSI_VECTOR_MAP1             0x58
 #define I2C_CH1_MSI_MAP_VECT_8      0x00000008
@@ -284,6 +293,8 @@ enum {
 #define MSI_VECTOR_REV_00           16
 #define MSI_VECTOR_REV_01           32
 
+#define FPGA_MSI_VECTOR_ID_4       4
+#define FPGA_MSI_VECTOR_ID_5       5
 #define FPGA_MSI_VECTOR_ID_8       8
 #define FPGA_MSI_VECTOR_ID_9       9
 #define FPGA_MSI_VECTOR_ID_10      10
@@ -506,6 +517,73 @@ static int fpgai2c_poll(struct fpgalogic_i2c *i2c)
     return 0;
 }
 
+static ssize_t get_mod_msi(struct device *dev, struct device_attribute *devattr, char *buf)
+{
+	int ind = 0, port_status=0, port_irq_status=0;
+    struct fpgapci_dev *fpgapci = (struct fpgapci_dev*) dev_get_drvdata(dev);
+	PRINT("%s:xcvr_intr_count:%u\n", __FUNCTION__, fpgapci->xcvr_intr_count);
+	for(ind=0;ind<64;ind++)
+	{ 
+       	port_status = ioread32(fpga_ctl_addr + PORT_STS_OFFSET + (ind*16));
+       	port_irq_status = ioread32(fpga_ctl_addr + PORT_IRQ_STS_OFFSET + (ind*16));
+		PRINT("%s:port:%d, port_status:%#x, port_irq_status:%#x\n", __FUNCTION__, ind, port_status, port_irq_status);
+	}	
+	return sprintf(buf,"0x%04x\n",fpgapci->xcvr_intr_count);
+}
+static DEVICE_ATTR(port_msi, S_IRUGO, get_mod_msi, NULL);
+
+static struct attribute *port_attrs[] = {
+    &dev_attr_port_msi.attr,
+    NULL,
+};
+
+static struct attribute_group port_attr_grp = {
+    .attrs = port_attrs,
+};
+
+
+static irqreturn_t fpgaport_1_32_isr(int irq, void *dev)
+{
+	struct pci_dev *pdev = dev;
+    struct fpgapci_dev *fpgapci = (struct fpgapci_dev*) dev_get_drvdata(&pdev->dev);
+	int ind = 0, port_status=0, port_irq_status=0;
+	for(ind=0;ind<32;ind++)
+	{ 
+        port_irq_status = ioread32(fpga_ctl_addr + PORT_IRQ_STS_OFFSET + (ind*16));
+		if(port_irq_status&(IRQ_LTCH_STS|PRSNT_LTCH_STS))
+		{
+			PRINT("%s:port:%d, port_status:%#x, port_irq_status:%#x\n", __FUNCTION__, ind, port_status, port_irq_status);
+				//write on clear
+        	iowrite32( IRQ_LTCH_STS|PRSNT_LTCH_STS,fpga_ctl_addr + PORT_IRQ_STS_OFFSET + (ind*16));
+		}
+	}	
+    fpgapci->xcvr_intr_count++;
+	PRINT("%s: xcvr_intr_count:%u\n", __FUNCTION__, fpgapci->xcvr_intr_count);
+	sysfs_notify(&pdev->dev.kobj, NULL, "port_msi");
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t fpgaport_33_64_isr(int irq, void *dev)
+{
+	struct pci_dev *pdev = dev;
+    struct fpgapci_dev *fpgapci = (struct fpgapci_dev*) dev_get_drvdata(&pdev->dev);
+	int ind = 0, port_status=0, port_irq_status=0;
+	for(ind=32;ind<64;ind++)
+	{ 
+        port_irq_status = ioread32(fpga_ctl_addr + PORT_IRQ_STS_OFFSET + (ind*16));
+		if(port_irq_status| (IRQ_LTCH_STS|PRSNT_LTCH_STS))
+		{
+			PRINT("%s:port:%d, port_status:%#x, port_irq_status:%#x\n", __FUNCTION__, ind, port_status, port_irq_status);
+				//write on clear
+        	iowrite32( IRQ_LTCH_STS|PRSNT_LTCH_STS,fpga_ctl_addr + PORT_IRQ_STS_OFFSET + (ind*16));
+		}
+	}	
+    fpgapci->xcvr_intr_count++;
+	PRINT("%s: xcvr_intr_count:%u\n", __FUNCTION__, fpgapci->xcvr_intr_count);
+	sysfs_notify(&pdev->dev.kobj, NULL, "port_msi");
+	return IRQ_HANDLED;
+}
+
 static void fpgai2c_process(struct fpgalogic_i2c *i2c)
 {
     struct i2c_msg *msg = i2c->msg;
@@ -1028,6 +1106,18 @@ static int register_intr_handler(struct pci_dev *dev, int irq_num_id)
         switch(irq_num_id) {
             /* Currently we only support test vector 2 for FPGA Logic I2C channel
              * controller 1-7  interrupt*/
+	    	case FPGA_MSI_VECTOR_ID_4: 
+                err = request_irq(dev->irq + irq_num_id, fpgaport_1_32_isr, IRQF_EARLY_RESUME,
+                        FPGA_PCI_NAME, dev);
+        		PRINT ( "%d: fpgapci_dev: irq: %d, %d\n", __LINE__, dev->irq, irq_num_id);
+                fpgapci->irq_assigned++;
+				break;
+	    	case FPGA_MSI_VECTOR_ID_5: 
+                err = request_irq(dev->irq + irq_num_id, fpgaport_33_64_isr, IRQF_EARLY_RESUME,
+                        FPGA_PCI_NAME, dev);
+        		PRINT ( "%d: fpgapci_dev: irq: %d, %d\n", __LINE__, dev->irq, irq_num_id);
+                fpgapci->irq_assigned++;
+				break;
             case FPGA_MSI_VECTOR_ID_8:
                 err = request_irq(dev->irq + irq_num_id, fpgai2c_isr, IRQF_EARLY_RESUME,
                         FPGA_PCI_NAME, &fpgalogic_i2c[0]);
@@ -1074,6 +1164,18 @@ static int register_intr_handler(struct pci_dev *dev, int irq_num_id)
             ((board_rev_type & MB_BRD_REV_MASK) == MB_BRD_REV_03)) {
         /* FPGA SPEC 4.3.1.34, First i2c channel mapped to vector 8 */
         switch (irq_num_id) {
+	    	case FPGA_MSI_VECTOR_ID_4: 
+                err = request_irq(dev->irq + irq_num_id, fpgaport_1_32_isr, IRQF_EARLY_RESUME,
+                        FPGA_PCI_NAME, dev);
+        		PRINT ( "%d: fpgapci_dev: irq: %d, %d\n", __LINE__, dev->irq, irq_num_id);
+                fpgapci->irq_assigned++;
+				break;
+	    	case FPGA_MSI_VECTOR_ID_5: 
+                err = request_irq(dev->irq + irq_num_id, fpgaport_33_64_isr, IRQF_EARLY_RESUME,
+                        FPGA_PCI_NAME, dev);
+        		PRINT ( "%d: fpgapci_dev: irq: %d, %d\n", __LINE__, dev->irq, irq_num_id);
+                fpgapci->irq_assigned++;
+				break;
             case FPGA_MSI_VECTOR_ID_8:
                 err = request_irq(dev->irq + irq_num_id, fpgai2c_isr, IRQF_EARLY_RESUME,
                         FPGA_PCI_NAME, &fpgalogic_i2c[0]);
@@ -1387,6 +1489,7 @@ error_no_msi:
 static int fpgapci_probe(struct pci_dev *dev, const struct pci_device_id *id)
 {
     struct fpgapci_dev *fpgapci = 0;
+    int status = 0;
 
 #ifdef TEST
     PRINT ( " vendor = 0x%x, device = 0x%x, class = 0x%x, bus:slot.func = %02x:%02x.%02x\n",
@@ -1404,6 +1507,10 @@ static int fpgapci_probe(struct pci_dev *dev, const struct pci_device_id *id)
     dev_set_drvdata(&dev->dev, (void*)fpgapci);
 
     fpgapci->upstream = find_upstream_dev (dev);
+    status = sysfs_create_group(&dev->dev.kobj, &port_attr_grp);
+    if (status) {
+        printk(KERN_INFO "%s:Cannot create sysfs\n", __FUNCTION__);
+    }
 
     if(fpgapci_setup_device(fpgapci,dev)) {
         goto error_no_device;

platform/broadcom/sonic-platform-modules-dell/z9264f/scripts/qsfp_irq_enable.py

@@ -0,0 +1,32 @@
+#!/usr/bin/python
+
+try:
+        import struct
+        import sys
+        from os import *
+        from mmap import *
+
+except ImportError as e:
+        raise ImportError("%s - required module no found" % str(e))
+
+BASE_RES_PATH = "/sys/bus/pci/devices/0000:04:00.0/resource0"
+PORT_START = 0
+PORT_END = 64
+
+
+def pci_mem_write(mm, offset, data):
+        mm.seek(offset)
+        mm.write(struct.pack('I', data))
+
+
+def pci_set_value(resource, val, offset):
+        fd = open(resource, O_RDWR)
+        mm = mmap(fd, 0)
+        val = pci_mem_write(mm, offset, val)
+        mm.close()
+        close(fd)
+        return val
+
+for port_num in range(PORT_START, PORT_END+1):
+        port_offset = 0x400c + ((port_num) * 16)
+        pci_set_value(BASE_RES_PATH, 0x30, port_offset)

platform/broadcom/sonic-platform-modules-dell/z9264f/scripts/z9264f_platform.sh

@@ -98,6 +98,7 @@ if [ "$1" == "init" ]; then
     switch_board_qsfp_mux "new_device"
     switch_board_qsfp "new_device"
     switch_board_modsel
+    python /usr/bin/qsfp_irq_enable.py
 
 elif [ "$1" == "deinit" ]; then
     sys_eeprom "delete_device"