#include #include #include #include #include #include #include #include #define NIC_STRING_LEN 1024 static bool __NIC_fillNicAddress(struct net_device* dev, NicAddrType_t nicType, NicAddress* outAddr); void NIC_findAll(StrCpyList* allowedInterfaces, bool useRDMA, bool onlyRDMA, NicAddressList* outList) { // find standard TCP/IP interfaces __NIC_findAllTCP(allowedInterfaces, outList); // find RDMA interfaces (based on TCP/IP interfaces query results) if(useRDMA && RDMASocket_rdmaDevicesExist() ) { NicAddressList tcpInterfaces; NicAddressList_init(&tcpInterfaces); __NIC_findAllTCP(allowedInterfaces, &tcpInterfaces); __NIC_filterInterfacesForRDMA(&tcpInterfaces, outList); ListTk_kfreeNicAddressListElems(&tcpInterfaces); NicAddressList_uninit(&tcpInterfaces); } if (onlyRDMA) { NicAddressListIter nicIter; NicAddressListIter_init(&nicIter, outList); while (!NicAddressListIter_end(&nicIter)) { NicAddress* nicAddr = NicAddressListIter_value(&nicIter); if (nicAddr->nicType != NICADDRTYPE_RDMA) { nicIter = NicAddressListIter_remove(&nicIter); kfree(nicAddr); } else NicAddressListIter_next(&nicIter); } } } void __NIC_findAllTCP(StrCpyList* allowedInterfaces, NicAddressList* outList) { struct net_device *dev; // find standard TCP/IP interfaces // foreach network device for (dev = first_net_device(&init_net); dev; dev = next_net_device(dev)) { NicAddress* nicAddr = (NicAddress*)os_kmalloc(sizeof(NicAddress) ); ssize_t metricByListPos = 0; if(!nicAddr) { printk_fhgfs(KERN_WARNING, "%s:%d: memory allocation failed. size: %zu\n", __func__, __LINE__, sizeof(*nicAddr) ); return; } if(__NIC_fillNicAddress(dev, NICADDRTYPE_STANDARD, nicAddr) && (!StrCpyList_length(allowedInterfaces) || ListTk_listContains(nicAddr->name, allowedInterfaces, &metricByListPos) ) ) { NicAddressList_append(outList, nicAddr); } else { // netdevice rejected => clean up kfree(nicAddr); } } } bool __NIC_fillNicAddress(struct net_device* dev, NicAddrType_t nicType, NicAddress* outAddr) { struct ifreq ifr; struct in_device* in_dev; struct in_ifaddr *ifa; #ifdef BEEGFS_RDMA outAddr->ibdev = NULL; #endif // name strncpy(outAddr->name, dev->name, IFNAMSIZ); // SIOCGIFFLAGS: // get interface flags ifr.ifr_flags = dev_get_flags(dev); if(ifr.ifr_flags & IFF_LOOPBACK) return false; // loopback interface => skip ifr.ifr_hwaddr.sa_family = dev->type; // select which hardware types to process // (on Linux see /usr/include/linux/if_arp.h for the whole list) switch(ifr.ifr_hwaddr.sa_family) { case ARPHRD_LOOPBACK: return false; default: break; } // copy nicType outAddr->nicType = nicType; // ip address // note: based on inet_gifconf in /net/ipv4/devinet.c in_dev = __in_dev_get_rtnl(dev); if(!in_dev) { printk_fhgfs_debug(KERN_NOTICE, "found interface without in_dev: %s\n", dev->name); return false; } ifa = in_dev->ifa_list; if(!ifa) { printk_fhgfs_debug(KERN_NOTICE, "found interface without ifa_list: %s\n", dev->name); return false; } outAddr->ipAddr.s_addr = ifa->ifa_local; // ip address // code to read multiple addresses /* for (; ifa; ifa = ifa->ifa_next) { (*(struct sockaddr_in *)&ifr.ifr_addr).sin_family = AF_INET; (*(struct sockaddr_in *)&ifr.ifr_addr).sin_addr.s_addr = ifa->ifa_local; } */ return true; } /** * @return static string (not alloc'ed, so don't free it). */ const char* NIC_nicTypeToString(NicAddrType_t nicType) { switch(nicType) { case NICADDRTYPE_RDMA: return "RDMA"; case NICADDRTYPE_STANDARD: return "TCP"; default: return ""; } } /** * @return string will be kalloced and must be kfreed later */ char* NIC_nicAddrToString(NicAddress* nicAddr) { char* nicAddrStr; char ipStr[NICADDRESS_IP_STR_LEN]; const char* typeStr; nicAddrStr = (char*)os_kmalloc(NIC_STRING_LEN); NicAddress_ipToStr(nicAddr->ipAddr, ipStr); if(nicAddr->nicType == NICADDRTYPE_RDMA) typeStr = "RDMA"; else if(nicAddr->nicType == NICADDRTYPE_STANDARD) typeStr = "TCP"; else typeStr = "Unknown"; snprintf(nicAddrStr, NIC_STRING_LEN, "%s[ip addr: %s; type: %s]", nicAddr->name, ipStr, typeStr); return nicAddrStr; } bool NIC_supportsRDMA(NicAddressList* nicList) { bool rdmaSupported = false; NicAddressListIter iter; NicAddressListIter_init(&iter, nicList); for( ; !NicAddressListIter_end(&iter); NicAddressListIter_next(&iter) ) { if(NicAddressListIter_value(&iter)->nicType == NICADDRTYPE_RDMA) { rdmaSupported = true; break; } } return rdmaSupported; } void NIC_supportedCapabilities(NicAddressList* nicList, NicListCapabilities* outCapabilities) { outCapabilities->supportsRDMA = NIC_supportsRDMA(nicList); } /** * Checks a list of TCP/IP interfaces for RDMA-capable interfaces. */ void __NIC_filterInterfacesForRDMA(NicAddressList* nicList, NicAddressList* outList) { // Note: This works by binding an RDMASocket to each IP of the passed list. NicAddressListIter iter; NicAddressListIter_init(&iter, nicList); for( ; !NicAddressListIter_end(&iter); NicAddressListIter_next(&iter) ) { RDMASocket rdmaSock; Socket* sock = (Socket*)&rdmaSock; NicAddress* nicAddr = NicAddressListIter_value(&iter); bool bindRes; if(!RDMASocket_init(&rdmaSock, nicAddr->ipAddr, NULL) ) continue; bindRes = sock->ops->bindToAddr(sock, nicAddr->ipAddr, 0); if(bindRes) { // we've got an RDMA-capable interface => append it to outList NicAddress* nicAddrCopy = os_kmalloc(sizeof(NicAddress) ); *nicAddrCopy = *nicAddr; #ifdef BEEGFS_RDMA nicAddrCopy->ibdev = rdmaSock.ibvsock.cm_id->device; #endif nicAddrCopy->nicType = NICADDRTYPE_RDMA; NicAddressList_append(outList, nicAddrCopy); } sock->ops->uninit(sock); } }