BXE: cleans up MMIO / BAR setup

Also, this sets up the second BAR, called BAR1 in the code, but it is actually
pcidev->bar[2], which is used for the doorbell.
diff --git a/kern/drivers/net/bxe/bxe.c b/kern/drivers/net/bxe/bxe.c
index 3f70e45..3ac18c7 100644
--- a/kern/drivers/net/bxe/bxe.c
+++ b/kern/drivers/net/bxe/bxe.c
@@ -13300,41 +13300,47 @@
 {
     unsigned int flags;
     int i;
+	physaddr_t mmio_paddr;
+	uint16_t pio_port;
     memset(sc->bar, 0, sizeof(sc->bar));
 
     for (i = 0; i < MAX_BARS; i++) {
 
         /* memory resources reside at BARs 0, 2, 4 */
         /* Run `pciconf -lb` to see mappings */
-        if ((i != 0) && (i != 2) && (i != 4)) {
-			/* i guess sc->bar[1] is just 0s */
+        if ((i != 0) && (i != 2) && (i != 4))
             continue;
-        }
 
         sc->bar[i].rid = PCIR_BAR(i);
 
-		/* The bar handles are supposed to be KVAs - they get dereferenced
-		 * later.  The addrs in the pcidev->bar are physical addrs.  For now,
-		 * we just map bar 0 and have it in sc->mmio. */
-		if (i == 0) {
-			sc->bar[i].tag = X86_BUS_SPACE_MEM;
-        	sc->bar[i].handle = (uintptr_t)sc->mmio;
+		mmio_paddr = pci_get_membar(sc->pcidev, i);
+		if (!mmio_paddr) {
+			pio_port = pci_get_iobar(sc->pcidev, i);
+			if (!pio_port) {
+				/* This will trip on BAR 4.  1 and 3 were skipped earlier, since
+				 * they are the second-parts of the MMIO64 addr.  Not sure why 4
+				 * gets skipped.  BSD seemed to think there was a memory
+				 * resource there. */
+				printk("BXE, skipping BAR %d\n", i);
+				continue;
+			}
+			sc->bar[i].handle = pio_port;
+			sc->bar[i].tag = X86_BUS_SPACE_IO;
+			printk("BXE: PIO mapped BAR %d, 0x%x\n", i, sc->bar[i].handle);
 		} else {
 			sc->bar[i].tag = X86_BUS_SPACE_MEM;
-        	sc->bar[i].handle = 0xcafeface;
+			sc->bar[i].handle = vmap_pmem_nocache(mmio_paddr,
+			                                      sc->pcidev->bar[i].mmio_sz);
+			if (!sc->bar[i].handle) {
+				printk("BXE: can't map %p for BAR %d\n", mmio_paddr, i);
+				continue;
+			}
+			printk("BXE: MMIO mapped BAR %d, %p -> %p\n", i, sc->bar[i].handle,
+			       mmio_paddr);
+			/* Here's how to remove it, if you need to: */
+			//vunmap_vmem(sc->bar[i].handle, sc->pcidev->bar[i].mmio_sz);
 		}
 
-		/* Maybe do something like this:
-        sc->bar[i].handle = pci_get_membar(sc->pcidev, i);
-		if (sc->bar[i].handle) {
-			sc->bar[i].tag = X86_BUS_SPACE_MEM;
-		} else {
-        	sc->bar[i].handle = pci_get_iobar(sc->pcidev, i);
-			if (sc->bar[i].handle) {
-				sc->bar[i].tag = X86_BUS_SPACE_IO;
-			}
-		}
-		*/
 
 #if 0 /* BSD way */
         flags = RF_ACTIVE;
diff --git a/kern/drivers/net/bxe/bxe.h b/kern/drivers/net/bxe/bxe.h
index b89e58b..d69a17b 100644
--- a/kern/drivers/net/bxe/bxe.h
+++ b/kern/drivers/net/bxe/bxe.h
@@ -1328,7 +1328,6 @@
 	struct ether				*edev;
 
 	bool						active;
-	void						*mmio;
 	spinlock_t					imlock;				/* interrupt mask lock */
 	spinlock_t					tlock;				/* transmit lock */
 	qlock_t						slock;				/* stats */
diff --git a/kern/drivers/net/bxe/bxe_dev.c b/kern/drivers/net/bxe/bxe_dev.c
index da0fbb0..21518c6 100644
--- a/kern/drivers/net/bxe/bxe_dev.c
+++ b/kern/drivers/net/bxe/bxe_dev.c
@@ -31,18 +31,6 @@
 	"Alignment Error",
 };
 
-/* Most 9ns drivers have some form of helper to read from the IO space, whether
- * that's PIO or MMIO. */
-static inline uint32_t csr32r(struct bxe_adapter *c, uintptr_t reg)
-{
-	return read_mmreg32((uintptr_t) (c->mmio + (reg / 4)));
-}
-
-static inline void csr32w(struct bxe_adapter *c, uintptr_t reg, uint32_t val)
-{
-	write_mmreg32((uintptr_t) (c->mmio + (reg / 4)), val);
-}
-
 static long bxeifstat(struct ether *edev, void *a, long n, uint32_t offset)
 {
 	struct bxe_adapter *ctlr;
@@ -329,8 +317,6 @@
 	int cls, id;
 	struct pci_device *pcidev;
 	struct bxe_adapter *ctlr;
-	void *mem;
-	uintptr_t mmio_paddr;
 
 	STAILQ_FOREACH(pcidev, &pci_devices, all_dev) {
 		/* This checks that pcidev is a Network Controller for Ethernet */
@@ -346,16 +332,8 @@
 			   pcidev->ven_id, pcidev->dev_id,
 			   pcidev->bus, pcidev->dev, pcidev->func);
 
-		/* Assuming MMIO */
-		/* Do this for each bar, based on whether it is mmio or not, and store
-		 * in the handles.  Move this to bxe_allocate_bars XME */
-		mmio_paddr = pci_get_membar(pcidev, 0);
-		assert(mmio_paddr);
-		mem = (void *)vmap_pmem_nocache(mmio_paddr, pcidev->bar[0].mmio_sz);
-		if (mem == NULL) {
-			printd("bxe: can't map %p\n", pcidev->bar[0].mmio_base32);
-			continue;
-		}
+		/* MMIO, pci_bus_master, etc, are all done in bxe_attach */
+
 		cls = pcidev_read8(pcidev, PCI_CLSZ_REG);
 		switch (cls) {
 			default:
@@ -374,10 +352,8 @@
 		}
 
 		ctlr = kzmalloc(sizeof(struct bxe_adapter), 0);
-		if (ctlr == NULL) {
-			vunmap_vmem((uintptr_t) mem, pcidev->bar[0].mmio_sz);
+		if (ctlr == NULL)
 			error(Enomem);
-		}
 
 		/* TODO: Remove me */
 		ctlr->debug = 0xFFFFFFFF; /* flying monkeys     */
@@ -389,17 +365,12 @@
 		rendez_init(&ctlr->rrendez);
 
 		ctlr->pcidev = pcidev;
-		ctlr->mmio = mem;
 		
 		if (bxereset(ctlr)) {
 			kfree(ctlr);
-			vunmap_vmem((uintptr_t) mem, pcidev->bar[0].mmio_sz);
 			continue;
 		}
 
-		/* this is done in bxe_attach too */ // XME
-		pci_set_bus_master(pcidev);
-
 		/* BSD used mutexes for this list for other reasons */
 		qlock(&bxe_prev_mtx);
 		LIST_INSERT_HEAD(&bxe_prev_list, ctlr, node);