From 33a500470dd0c9f0d7c0a88ea70c991c7aa0a718 Mon Sep 17 00:00:00 2001 From: kram Date: Thu, 9 Nov 2017 18:33:16 -0800 Subject: [PATCH] revert core creation setting --- dockers/docker-orchagent/orchagent.sh | 3 - .../modules/bf_kdrv.c | 193 +++++++++++++----- 2 files changed, 145 insertions(+), 51 deletions(-) diff --git a/dockers/docker-orchagent/orchagent.sh b/dockers/docker-orchagent/orchagent.sh index 587909cd20..2bec34951e 100755 --- a/dockers/docker-orchagent/orchagent.sh +++ b/dockers/docker-orchagent/orchagent.sh @@ -1,8 +1,5 @@ #!/usr/bin/env bash -ulimit -c unlimited -sysctl -w kernel.core_pattern=core - # Export platform information. Required to be able to write # vendor specific code. export platform=`sonic-cfggen -y /etc/sonic/sonic_version.yml -v asic_type` diff --git a/platform/barefoot/sonic-platform-modules-bfn/modules/bf_kdrv.c b/platform/barefoot/sonic-platform-modules-bfn/modules/bf_kdrv.c index f486a41644..5f084f6a04 100644 --- a/platform/barefoot/sonic-platform-modules-bfn/modules/bf_kdrv.c +++ b/platform/barefoot/sonic-platform-modules-bfn/modules/bf_kdrv.c @@ -61,6 +61,7 @@ #include #include #include +#include #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0) //#error unsupported linux kernel version @@ -71,7 +72,8 @@ extern int pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec); extern int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries, int minvec, int maxvec); #define PCI_VENDOR_ID_BF 0x1d1c -#define TOFINO_DEV_ID 1 +#define TOFINO_DEV_ID_A0 0x01 +#define TOFINO_DEV_ID_B0 0x10 #ifndef PCI_MSIX_ENTRY_SIZE #define PCI_MSIX_ENTRY_SIZE 16 @@ -108,13 +110,19 @@ struct bf_dev_mem { void __iomem *internal_addr; }; +struct bf_listener { + struct bf_pci_dev *bfdev; + s32 event_count[BF_MSIX_ENTRY_CNT]; + int minor; + struct bf_listener *next; +}; + /* device information */ struct bf_dev_info { struct module *owner; struct device *dev; int minor; atomic_t event[BF_MSIX_ENTRY_CNT]; - struct fasync_struct *async_queue; wait_queue_head_t wait; const char *version; struct bf_dev_mem mem[BF_MAX_BAR_MAPS]; @@ -138,11 +146,21 @@ struct bf_int_vector { struct bf_pci_dev { struct bf_dev_info info; struct pci_dev *pdev; - struct cdev cdev; enum bf_intr_mode mode; u8 instance; char name[16]; struct bf_int_vector bf_int_vec[BF_MSIX_ENTRY_CNT]; + struct bf_listener *listener_head; /* head of a singly linked list of + listeners */ +}; + +/* Keep any global information here that must survive even after the + * bf_pci_dev is free-ed up. + */ +struct bf_global { + struct bf_pci_dev *bfdev ; + struct cdev *bf_cdev; + struct fasync_struct *async_queue; }; static int bf_major; @@ -151,6 +169,55 @@ static struct class *bf_class = NULL; static char *intr_mode = NULL; static enum bf_intr_mode bf_intr_mode_default = BF_INTR_MODE_MSI; static spinlock_t bf_nonisr_lock; +/* dev->minor should index into this array */ +static struct bf_global bf_global[BF_MAX_DEVICE_CNT]; + +static void bf_add_listener(struct bf_pci_dev *bfdev, + struct bf_listener *listener) +{ + struct bf_listener **cur_listener = &bfdev->listener_head; + + if (!listener) { + return; + } + spin_lock(&bf_nonisr_lock); + + while (*cur_listener) { + cur_listener = &((*cur_listener)->next); + } + *cur_listener = listener; + listener->next = NULL; + + spin_unlock(&bf_nonisr_lock); +} + +static void bf_remove_listener(struct bf_pci_dev *bfdev, + struct bf_listener *listener) +{ + struct bf_listener **cur_listener = &bfdev->listener_head; + + /* in case of certain error conditions, this function might be called after bf_pci_remove() + */ + if (!bfdev || !listener) { + return; + } + spin_lock(&bf_nonisr_lock); + + if (*cur_listener == listener) { + *cur_listener = listener->next; + } else { + while (*cur_listener) { + if ((*cur_listener)->next == listener) { + (*cur_listener)->next = listener->next; + break; + } + cur_listener = &((*cur_listener)->next); + } + listener->next = NULL; + } + + spin_unlock(&bf_nonisr_lock); +} /* a pool of minor numbers is maintained */ /* return the first available minor number */ @@ -345,17 +412,15 @@ static irqreturn_t bf_interrupt(int irq, void *bfdev_id) return ret; } -struct bf_listener { - struct bf_pci_dev *bfdev; - s32 event_count[BF_MSIX_ENTRY_CNT]; -}; - static unsigned int bf_poll(struct file *filep, poll_table *wait) { struct bf_listener *listener = (struct bf_listener *)filep->private_data; struct bf_pci_dev *bfdev = listener->bfdev; int i; + if (!bfdev) { + return -ENODEV; + } if (!bfdev->info.irq) return -EIO; @@ -422,6 +487,9 @@ static int bf_mmap(struct file *filep, struct vm_area_struct *vma) int bar; unsigned long requested_pages, actual_pages; + if (!bfdev) { + return -ENODEV; + } if (vma->vm_end < vma->vm_start) return -EINVAL; @@ -442,12 +510,19 @@ static int bf_mmap(struct file *filep, struct vm_area_struct *vma) static int bf_fasync(int fd, struct file *filep, int mode) { - struct bf_pci_dev *bfdev = ((struct bf_listener *)filep->private_data)->bfdev; + int minor; - if (mode == 0 && &bfdev->info.async_queue == NULL) { + if (!filep->private_data) { + return (-EINVAL); + } + minor = ((struct bf_listener *)filep->private_data)->minor; + if (minor >= BF_MAX_DEVICE_CNT) { + return (-EINVAL); + } + if (mode == 0 && &bf_global[minor].async_queue == NULL) { return 0; /* nothing to do */ } - return (fasync_helper(fd, filep, mode, &bfdev->info.async_queue)); + return (fasync_helper(fd, filep, mode, &bf_global[minor].async_queue)); } static int bf_open(struct inode *inode, struct file *filep) @@ -456,10 +531,13 @@ static int bf_open(struct inode *inode, struct file *filep) struct bf_listener *listener; int i; - bfdev = container_of(inode->i_cdev, struct bf_pci_dev, cdev); + bfdev = bf_global[iminor(inode)].bfdev; listener = kmalloc(sizeof(*listener), GFP_KERNEL); if (listener) { listener->bfdev = bfdev; + listener->minor = bfdev->info.minor; + listener->next = NULL; + bf_add_listener(bfdev, listener); for (i = 0; i < BF_MSIX_ENTRY_CNT; i++) listener->event_count[i] = atomic_read(&bfdev->info.event[i]); filep->private_data = listener; @@ -474,6 +552,9 @@ static int bf_release(struct inode *inode, struct file *filep) struct bf_listener *listener = filep->private_data; bf_fasync(-1, filep, 0); /* empty any process id in the notification list */ + if (listener->bfdev) { + bf_remove_listener(listener->bfdev, listener); + } kfree(listener); return 0; } @@ -488,6 +569,9 @@ static ssize_t bf_read(struct file *filep, char __user *buf, int i, mismatch_found = 0; /* OR of per vector mismatch */ unsigned char cnt_match[BF_MSIX_ENTRY_CNT]; /* per vector mismatch */ + if (!bfdev) { + return -ENODEV; + } /* irq must be setup for read() to work */ if (!bfdev->info.irq) return -EIO; @@ -593,8 +677,9 @@ static const struct file_operations bf_fops = { .fasync = bf_fasync, }; -static int bf_major_init(struct bf_pci_dev *bfdev) +static int bf_major_init(struct bf_pci_dev *bfdev, int minor) { + struct cdev *cdev; static const char name[] = "bf"; dev_t bf_dev = 0; int result; @@ -604,26 +689,20 @@ static int bf_major_init(struct bf_pci_dev *bfdev) return result; result = -ENOMEM; -/* following code useful if we chose to allocate cdev here - * - *struct cdev *cdev = NULL; - * cdev = cdev_alloc(); - * if (!cdev) - * goto fail_unregister; - * cdev->ops = &bf_fops; - * cdev->owner = THIS_MODULE; - * kobject_set_name(&cdev->kobj, "%s", name); - * result = cdev_add(cdev, bf_dev, BF_MAX_DEVICE_CNT); - */ - cdev_init(&bfdev->cdev, &bf_fops); - bfdev->cdev.owner = THIS_MODULE; - bfdev->cdev.ops = &bf_fops; + cdev = cdev_alloc(); + if (!cdev) { + goto fail_dev_add; + } + cdev->ops = &bf_fops; + cdev->owner = THIS_MODULE; + kobject_set_name(&cdev->kobj, "%s", name); + result = cdev_add(cdev, bf_dev, BF_MAX_DEVICE_CNT); - result = cdev_add(&bfdev->cdev, bf_dev, BF_MAX_DEVICE_CNT); if (result) goto fail_dev_add; bf_major = MAJOR(bf_dev); + bf_global[minor].bf_cdev = cdev; return 0; fail_dev_add: @@ -631,16 +710,16 @@ fail_dev_add: return result; } -static void bf_major_cleanup(struct bf_pci_dev *bfdev) +static void bf_major_cleanup(struct bf_pci_dev *bfdev, int minor) { unregister_chrdev_region(MKDEV(bf_major, 0), BF_MAX_DEVICE_CNT); - cdev_del(&bfdev->cdev); + cdev_del(bf_global[minor].bf_cdev); } -static int bf_init_cdev(struct bf_pci_dev *bfdev) +static int bf_init_cdev(struct bf_pci_dev *bfdev, int minor) { int ret; - ret = bf_major_init(bfdev); + ret = bf_major_init(bfdev, minor); if (ret) return ret; @@ -653,14 +732,14 @@ static int bf_init_cdev(struct bf_pci_dev *bfdev) return 0; err_class_register: - bf_major_cleanup(bfdev); + bf_major_cleanup(bfdev, minor); return ret; } static void bf_remove_cdev(struct bf_pci_dev *bfdev) { class_destroy(bf_class); - bf_major_cleanup(bfdev); + bf_major_cleanup(bfdev, bfdev->info.minor); } @@ -686,16 +765,16 @@ int bf_register_device(struct device *parent, struct bf_pci_dev *bfdev) atomic_set(&info->event[i], 0); } - ret = bf_init_cdev(bfdev); + if (bf_get_next_minor_no(&minor)) { + return -EINVAL; + } + + ret = bf_init_cdev(bfdev, minor); if (ret) { printk(KERN_ERR "BF: device cdev creation failed\n"); return ret; } - if (bf_get_next_minor_no(&minor)) { - return -EINVAL; - } - info->dev = device_create(bf_class, parent, MKDEV(bf_major, minor), bfdev, "bf%d", minor); @@ -806,6 +885,8 @@ bf_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) int err, pci_use_highmem; int i, num_irq; + memset(bf_global, 0, sizeof(bf_global)); + bfdev = kzalloc(sizeof(struct bf_pci_dev), GFP_KERNEL); if (!bfdev) return -ENOMEM; @@ -816,8 +897,6 @@ bf_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) bfdev->bf_int_vec[i].bf_dev = bfdev; } - bfdev->info.async_queue = NULL; - /* initialize intr_mode to none */ bfdev->mode = BF_INTR_MODE_NONE; @@ -953,12 +1032,15 @@ bf_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto fail_clear_pci_master; } + pci_set_drvdata(pdev, bfdev); + sprintf(bfdev->name, "bf_%d", bfdev->info.minor); /* register bf driver */ err = bf_register_device(&pdev->dev, bfdev); if (err != 0) goto fail_release_irq; - sprintf(bfdev->name, "bf_%d", bfdev->info.minor); - pci_set_drvdata(pdev, bfdev); + + bf_global[bfdev->info.minor].async_queue = NULL; + bf_global[bfdev->info.minor].bfdev = bfdev; dev_info(&pdev->dev, "bf device %d registered with irq %ld\n", bfdev->instance, bfdev->info.irq); @@ -966,6 +1048,7 @@ bf_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) return 0; fail_release_irq: + pci_set_drvdata(pdev, NULL); if (bfdev->mode == BF_INTR_MODE_MSIX) { pci_disable_msix(bfdev->pdev); kfree(bfdev->info.msix_entries); @@ -991,6 +1074,7 @@ static void bf_pci_remove(struct pci_dev *pdev) { struct bf_pci_dev *bfdev = pci_get_drvdata(pdev); + struct bf_listener *cur_listener; bf_unregister_device(bfdev); if (bfdev->mode == BF_INTR_MODE_MSIX) { @@ -1006,6 +1090,16 @@ bf_pci_remove(struct pci_dev *pdev) pci_disable_pcie_error_reporting(pdev); pci_disable_device(pdev); pci_set_drvdata(pdev, NULL); + bf_global[bfdev->info.minor].bfdev = NULL; + /* existing filep structures in open file(s) must be informed that + * bf_pci_dev is no longer valid */ + spin_lock(&bf_nonisr_lock); + cur_listener = bfdev->listener_head; + while (cur_listener) { + cur_listener->bfdev = NULL; + cur_listener = cur_listener->next; + } + spin_unlock(&bf_nonisr_lock); kfree(bfdev); } @@ -1020,16 +1114,18 @@ static pci_ers_result_t bf_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state) { struct bf_pci_dev *bfdev = pci_get_drvdata(pdev); + int minor; if (!bfdev) { return PCI_ERS_RESULT_NONE; } - printk(KERN_ERR "pci_err_detected state %d asyncQ %p\n", state, &bfdev->info.async_queue); + printk(KERN_ERR "pci_err_detected state %d\n", state); if (state == pci_channel_io_perm_failure || state == pci_channel_io_frozen) { bfdev->info.pci_error_state = 1; /* send a signal to the user space program of the error */ - if (bfdev->info.async_queue) { - kill_fasync(&bfdev->info.async_queue, SIGIO, POLL_ERR); + minor = bfdev->info.minor; + if (minor < BF_MAX_DEVICE_CNT && bf_global[minor].async_queue) { + kill_fasync(&bf_global[minor].async_queue, SIGIO, POLL_ERR); } return PCI_ERS_RESULT_DISCONNECT; } else { @@ -1096,7 +1192,8 @@ bf_config_intr_mode(char *intr_str) } static const struct pci_device_id bf_pci_tbl[] = { - {PCI_VDEVICE(BF, TOFINO_DEV_ID), 0}, + {PCI_VDEVICE(BF, TOFINO_DEV_ID_A0), 0}, + {PCI_VDEVICE(BF, TOFINO_DEV_ID_B0), 0}, /* required last entry */ { .device = 0 } }; @@ -1140,7 +1237,7 @@ module_exit(bfdrv_exit); module_param(intr_mode, charp, S_IRUGO); MODULE_PARM_DESC(intr_mode, -"bf interrupt mode (default=msi):\n" +"bf interrupt mode (default=msix):\n" " " BF_INTR_MODE_MSIX_NAME " Use MSIX interrupt\n" " " BF_INTR_MODE_MSI_NAME " Use MSI interrupt\n" " " BF_INTR_MODE_LEGACY_NAME " Use Legacy interrupt\n"