748 	struct ipoib_neigh *neigh, *tn;  in path_rec_completion()  local
804 		list_for_each_entry_safe(neigh, tn, &path->neigh_list, list) {  in path_rec_completion()
805 			if (neigh->ah) {  in path_rec_completion()
806 				WARN_ON(neigh->ah != old_ah);  in path_rec_completion()
814 				ipoib_put_ah(neigh->ah);  in path_rec_completion()
817 			neigh->ah = path->ah;  in path_rec_completion()
819 			if (ipoib_cm_enabled(dev, neigh->daddr)) {  in path_rec_completion()
820 				if (!ipoib_cm_get(neigh))  in path_rec_completion()
821 					ipoib_cm_set(neigh, ipoib_cm_create_tx(dev,  in path_rec_completion()
823 									       neigh));  in path_rec_completion()
824 				if (!ipoib_cm_get(neigh)) {  in path_rec_completion()
825 					ipoib_neigh_free(neigh);  in path_rec_completion()
830 			while ((skb = __skb_dequeue(&neigh->queue)))  in path_rec_completion()
926 static void neigh_refresh_path(struct ipoib_neigh *neigh, u8 *daddr,  in neigh_refresh_path()  argument
950 	struct ipoib_neigh *neigh;  in neigh_add_path()  local
954 	neigh = ipoib_neigh_alloc(daddr, dev);  in neigh_add_path()
955 	if (!neigh) {  in neigh_add_path()
965 	if (unlikely(!list_empty(&neigh->list))) {  in neigh_add_path()
967 		return neigh;  in neigh_add_path()
979 	list_add_tail(&neigh->list, &path->neigh_list);  in neigh_add_path()
983 		neigh->ah = path->ah;  in neigh_add_path()
985 		if (ipoib_cm_enabled(dev, neigh->daddr)) {  in neigh_add_path()
986 			if (!ipoib_cm_get(neigh))  in neigh_add_path()
987 				ipoib_cm_set(neigh, ipoib_cm_create_tx(dev, path, neigh));  in neigh_add_path()
988 			if (!ipoib_cm_get(neigh)) {  in neigh_add_path()
989 				ipoib_neigh_free(neigh);  in neigh_add_path()
992 			if (skb_queue_len(&neigh->queue) <  in neigh_add_path()
994 				push_pseudo_header(skb, neigh->daddr);  in neigh_add_path()
995 				__skb_queue_tail(&neigh->queue, skb);  in neigh_add_path()
998 					   skb_queue_len(&neigh->queue));  in neigh_add_path()
1005 			ipoib_neigh_put(neigh);  in neigh_add_path()
1009 		neigh->ah  = NULL;  in neigh_add_path()
1013 		if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) {  in neigh_add_path()
1014 			push_pseudo_header(skb, neigh->daddr);  in neigh_add_path()
1015 			__skb_queue_tail(&neigh->queue, skb);  in neigh_add_path()
1022 	ipoib_neigh_put(neigh);  in neigh_add_path()
1026 	ipoib_neigh_free(neigh);  in neigh_add_path()
1032 	ipoib_neigh_put(neigh);  in neigh_add_path()
1096 	struct ipoib_neigh *neigh;  in ipoib_start_xmit()  local
1121 		neigh = ipoib_neigh_get(dev, phdr->hwaddr);  in ipoib_start_xmit()
1122 		if (likely(neigh))  in ipoib_start_xmit()
1133 		neigh = ipoib_neigh_get(dev, phdr->hwaddr);  in ipoib_start_xmit()
1134 		if (unlikely(!neigh)) {  in ipoib_start_xmit()
1135 			neigh = neigh_add_path(skb, phdr->hwaddr, dev);  in ipoib_start_xmit()
1136 			if (likely(!neigh))  in ipoib_start_xmit()
1154 	if (ipoib_cm_get(neigh)) {  in ipoib_start_xmit()
1155 		if (ipoib_cm_up(neigh)) {  in ipoib_start_xmit()
1156 			ipoib_cm_send(dev, skb, ipoib_cm_get(neigh));  in ipoib_start_xmit()
1159 	} else if (neigh->ah && neigh->ah->valid) {  in ipoib_start_xmit()
1160 		neigh->ah->last_send = rn->send(dev, skb, neigh->ah->ah,  in ipoib_start_xmit()
1163 	} else if (neigh->ah) {  in ipoib_start_xmit()
1164 		neigh_refresh_path(neigh, phdr->hwaddr, dev);  in ipoib_start_xmit()
1167 	if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) {  in ipoib_start_xmit()
1170 		__skb_queue_tail(&neigh->queue, skb);  in ipoib_start_xmit()
1178 	ipoib_neigh_put(neigh);  in ipoib_start_xmit()
1266 	struct ipoib_neigh *neigh = NULL;  in ipoib_neigh_get()  local
1277 	for (neigh = rcu_dereference_bh(htbl->buckets[hash_val]);  in ipoib_neigh_get()
1278 	     neigh != NULL;  in ipoib_neigh_get()
1279 	     neigh = rcu_dereference_bh(neigh->hnext)) {  in ipoib_neigh_get()
1280 		if (memcmp(daddr, neigh->daddr, INFINIBAND_ALEN) == 0) {  in ipoib_neigh_get()
1282 			if (!atomic_inc_not_zero(&neigh->refcnt)) {  in ipoib_neigh_get()
1284 				neigh = NULL;  in ipoib_neigh_get()
1288 			if (likely(skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE))  in ipoib_neigh_get()
1289 				neigh->alive = jiffies;  in ipoib_neigh_get()
1296 	return neigh;  in ipoib_neigh_get()
1322 		struct ipoib_neigh *neigh;  in __ipoib_reap_neigh()  local
1325 		while ((neigh = rcu_dereference_protected(*np,  in __ipoib_reap_neigh()
1328 			if (time_after(neigh_obsolete, neigh->alive)) {  in __ipoib_reap_neigh()
1330 				ipoib_check_and_add_mcast_sendonly(priv, neigh->daddr + 4, &remove_list);  in __ipoib_reap_neigh()
1333 						   rcu_dereference_protected(neigh->hnext,  in __ipoib_reap_neigh()
1336 				list_del_init(&neigh->list);  in __ipoib_reap_neigh()
1337 				call_rcu(&neigh->rcu, ipoib_neigh_reclaim);  in __ipoib_reap_neigh()
1339 				np = &neigh->hnext;  in __ipoib_reap_neigh()
1365 	struct ipoib_neigh *neigh;  in ipoib_neigh_ctor()  local
1367 	neigh = kzalloc(sizeof(*neigh), GFP_ATOMIC);  in ipoib_neigh_ctor()
1368 	if (!neigh)  in ipoib_neigh_ctor()
1371 	neigh->dev = dev;  in ipoib_neigh_ctor()
1372 	memcpy(&neigh->daddr, daddr, sizeof(neigh->daddr));  in ipoib_neigh_ctor()
1373 	skb_queue_head_init(&neigh->queue);  in ipoib_neigh_ctor()
1374 	INIT_LIST_HEAD(&neigh->list);  in ipoib_neigh_ctor()
1375 	ipoib_cm_set(neigh, NULL);  in ipoib_neigh_ctor()
1377 	atomic_set(&neigh->refcnt, 1);  in ipoib_neigh_ctor()
1379 	return neigh;  in ipoib_neigh_ctor()
1388 	struct ipoib_neigh *neigh;  in ipoib_neigh_alloc()  local
1394 		neigh = NULL;  in ipoib_neigh_alloc()
1402 	for (neigh = rcu_dereference_protected(htbl->buckets[hash_val],  in ipoib_neigh_alloc()
1404 	     neigh != NULL;  in ipoib_neigh_alloc()
1405 	     neigh = rcu_dereference_protected(neigh->hnext,  in ipoib_neigh_alloc()
1407 		if (memcmp(daddr, neigh->daddr, INFINIBAND_ALEN) == 0) {  in ipoib_neigh_alloc()
1409 			if (!atomic_inc_not_zero(&neigh->refcnt)) {  in ipoib_neigh_alloc()
1411 				neigh = NULL;  in ipoib_neigh_alloc()
1414 			neigh->alive = jiffies;  in ipoib_neigh_alloc()
1419 	neigh = ipoib_neigh_ctor(daddr, dev);  in ipoib_neigh_alloc()
1420 	if (!neigh)  in ipoib_neigh_alloc()
1424 	atomic_inc(&neigh->refcnt);  in ipoib_neigh_alloc()
1425 	neigh->alive = jiffies;  in ipoib_neigh_alloc()
1427 	rcu_assign_pointer(neigh->hnext,  in ipoib_neigh_alloc()
1430 	rcu_assign_pointer(htbl->buckets[hash_val], neigh);  in ipoib_neigh_alloc()
1435 	return neigh;  in ipoib_neigh_alloc()
1438 void ipoib_neigh_dtor(struct ipoib_neigh *neigh)  in ipoib_neigh_dtor()  argument
1441 	struct net_device *dev = neigh->dev;  in ipoib_neigh_dtor()
1444 	if (neigh->ah)  in ipoib_neigh_dtor()
1445 		ipoib_put_ah(neigh->ah);  in ipoib_neigh_dtor()
1446 	while ((skb = __skb_dequeue(&neigh->queue))) {  in ipoib_neigh_dtor()
1450 	if (ipoib_cm_get(neigh))  in ipoib_neigh_dtor()
1451 		ipoib_cm_destroy_tx(ipoib_cm_get(neigh));  in ipoib_neigh_dtor()
1454 		  IPOIB_QPN(neigh->daddr),  in ipoib_neigh_dtor()
1455 		  neigh->daddr + 4);  in ipoib_neigh_dtor()
1456 	kfree(neigh);  in ipoib_neigh_dtor()
1466 	struct ipoib_neigh *neigh = container_of(rp, struct ipoib_neigh, rcu);  in ipoib_neigh_reclaim()  local
1468 	ipoib_neigh_put(neigh);  in ipoib_neigh_reclaim()
1471 void ipoib_neigh_free(struct ipoib_neigh *neigh)  in ipoib_neigh_free()  argument
1473 	struct net_device *dev = neigh->dev;  in ipoib_neigh_free()
1486 	hash_val = ipoib_addr_hash(htbl, neigh->daddr);  in ipoib_neigh_free()
1493 		if (n == neigh) {  in ipoib_neigh_free()
1496 					   rcu_dereference_protected(neigh->hnext,  in ipoib_neigh_free()
1499 			list_del_init(&neigh->list);  in ipoib_neigh_free()
1500 			call_rcu(&neigh->rcu, ipoib_neigh_reclaim);  in ipoib_neigh_free()
1571 		struct ipoib_neigh *neigh;  in ipoib_del_neighs_by_gid()  local
1574 		while ((neigh = rcu_dereference_protected(*np,  in ipoib_del_neighs_by_gid()
1577 			if (!memcmp(gid, neigh->daddr + 4, sizeof (union ib_gid))) {  in ipoib_del_neighs_by_gid()
1579 						   rcu_dereference_protected(neigh->hnext,  in ipoib_del_neighs_by_gid()
1582 				list_del_init(&neigh->list);  in ipoib_del_neighs_by_gid()
1583 				call_rcu(&neigh->rcu, ipoib_neigh_reclaim);  in ipoib_del_neighs_by_gid()
1585 				np = &neigh->hnext;  in ipoib_del_neighs_by_gid()
1616 		struct ipoib_neigh *neigh;  in ipoib_flush_neighs()  local
1619 		while ((neigh = rcu_dereference_protected(*np,  in ipoib_flush_neighs()
1622 					   rcu_dereference_protected(neigh->hnext,  in ipoib_flush_neighs()
1625 			list_del_init(&neigh->list);  in ipoib_flush_neighs()
1626 			call_rcu(&neigh->rcu, ipoib_neigh_reclaim);  in ipoib_flush_neighs()