548 static int noop_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,  in noop_enqueue()  argument
555 static struct sk_buff *noop_dequeue(struct Qdisc *qdisc)  in noop_dequeue()  argument
570 	RCU_POINTER_INITIALIZER(qdisc, &noop_qdisc),
598 static int noqueue_init(struct Qdisc *qdisc, struct nlattr *opt,  in noqueue_init()  argument
604 	qdisc->enqueue = NULL;  in noqueue_init()
642 static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,  in pfifo_fast_enqueue()  argument
646 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);  in pfifo_fast_enqueue()
654 		if (qdisc_is_percpu_stats(qdisc))  in pfifo_fast_enqueue()
655 			return qdisc_drop_cpu(skb, qdisc, to_free);  in pfifo_fast_enqueue()
657 			return qdisc_drop(skb, qdisc, to_free);  in pfifo_fast_enqueue()
660 	qdisc_update_stats_at_enqueue(qdisc, pkt_len);  in pfifo_fast_enqueue()
664 static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)  in pfifo_fast_dequeue()  argument
666 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);  in pfifo_fast_dequeue()
681 		qdisc_update_stats_at_dequeue(qdisc, skb);  in pfifo_fast_dequeue()
683 		   test_bit(__QDISC_STATE_MISSED, &qdisc->state)) {  in pfifo_fast_dequeue()
689 		clear_bit(__QDISC_STATE_MISSED, &qdisc->state);  in pfifo_fast_dequeue()
700 		WRITE_ONCE(qdisc->empty, true);  in pfifo_fast_dequeue()
706 static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc)  in pfifo_fast_peek()  argument
708 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);  in pfifo_fast_peek()
721 static void pfifo_fast_reset(struct Qdisc *qdisc)  in pfifo_fast_reset()  argument
724 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);  in pfifo_fast_reset()
740 	if (qdisc_is_percpu_stats(qdisc)) {  in pfifo_fast_reset()
744 			q = per_cpu_ptr(qdisc->cpu_qstats, i);  in pfifo_fast_reset()
751 static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)  in pfifo_fast_dump()  argument
764 static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt,  in pfifo_fast_init()  argument
767 	unsigned int qlen = qdisc_dev(qdisc)->tx_queue_len;  in pfifo_fast_init()
768 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);  in pfifo_fast_init()
785 	qdisc->flags |= TCQ_F_CAN_BYPASS;  in pfifo_fast_init()
943 void qdisc_reset(struct Qdisc *qdisc)  in qdisc_reset()  argument
945 	const struct Qdisc_ops *ops = qdisc->ops;  in qdisc_reset()
948 	trace_qdisc_reset(qdisc);  in qdisc_reset()
951 		ops->reset(qdisc);  in qdisc_reset()
953 	skb_queue_walk_safe(&qdisc->gso_skb, skb, tmp) {  in qdisc_reset()
954 		__skb_unlink(skb, &qdisc->gso_skb);  in qdisc_reset()
958 	skb_queue_walk_safe(&qdisc->skb_bad_txq, skb, tmp) {  in qdisc_reset()
959 		__skb_unlink(skb, &qdisc->skb_bad_txq);  in qdisc_reset()
963 	qdisc->q.qlen = 0;  in qdisc_reset()
964 	qdisc->qstats.backlog = 0;  in qdisc_reset()
968 void qdisc_free(struct Qdisc *qdisc)  in qdisc_free()  argument
970 	if (qdisc_is_percpu_stats(qdisc)) {  in qdisc_free()
971 		free_percpu(qdisc->cpu_bstats);  in qdisc_free()
972 		free_percpu(qdisc->cpu_qstats);  in qdisc_free()
975 	kfree(qdisc);  in qdisc_free()
985 static void qdisc_destroy(struct Qdisc *qdisc)  in qdisc_destroy()  argument
987 	const struct Qdisc_ops  *ops = qdisc->ops;  in qdisc_destroy()
990 	qdisc_hash_del(qdisc);  in qdisc_destroy()
992 	qdisc_put_stab(rtnl_dereference(qdisc->stab));  in qdisc_destroy()
994 	gen_kill_estimator(&qdisc->rate_est);  in qdisc_destroy()
996 	qdisc_reset(qdisc);  in qdisc_destroy()
999 		ops->destroy(qdisc);  in qdisc_destroy()
1002 	dev_put(qdisc_dev(qdisc));  in qdisc_destroy()
1004 	trace_qdisc_destroy(qdisc);  in qdisc_destroy()
1006 	call_rcu(&qdisc->rcu, qdisc_free_cb);  in qdisc_destroy()
1009 void qdisc_put(struct Qdisc *qdisc)  in qdisc_put()  argument
1011 	if (!qdisc)  in qdisc_put()
1014 	if (qdisc->flags & TCQ_F_BUILTIN ||  in qdisc_put()
1015 	    !refcount_dec_and_test(&qdisc->refcnt))  in qdisc_put()
1018 	qdisc_destroy(qdisc);  in qdisc_put()
1027 void qdisc_put_unlocked(struct Qdisc *qdisc)  in qdisc_put_unlocked()  argument
1029 	if (qdisc->flags & TCQ_F_BUILTIN ||  in qdisc_put_unlocked()
1030 	    !refcount_dec_and_rtnl_lock(&qdisc->refcnt))  in qdisc_put_unlocked()
1033 	qdisc_destroy(qdisc);  in qdisc_put_unlocked()
1040 			      struct Qdisc *qdisc)  in dev_graft_qdisc()  argument
1049 	if (qdisc == NULL)  in dev_graft_qdisc()
1050 		qdisc = &noop_qdisc;  in dev_graft_qdisc()
1051 	dev_queue->qdisc_sleeping = qdisc;  in dev_graft_qdisc()
1052 	rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);  in dev_graft_qdisc()
1064 	struct Qdisc *qdisc = dev_queue->qdisc_sleeping;  in shutdown_scheduler_queue()  local
1067 	if (qdisc) {  in shutdown_scheduler_queue()
1068 		rcu_assign_pointer(dev_queue->qdisc, qdisc_default);  in shutdown_scheduler_queue()
1071 		qdisc_put(qdisc);  in shutdown_scheduler_queue()
1079 	struct Qdisc *qdisc;  in attach_one_default_qdisc()  local
1087 	qdisc = qdisc_create_dflt(dev_queue, ops, TC_H_ROOT, NULL);  in attach_one_default_qdisc()
1088 	if (!qdisc)  in attach_one_default_qdisc()
1092 		qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;  in attach_one_default_qdisc()
1093 	dev_queue->qdisc_sleeping = qdisc;  in attach_one_default_qdisc()
1099 	struct Qdisc *qdisc;  in attach_default_qdiscs()  local
1106 		qdisc = txq->qdisc_sleeping;  in attach_default_qdiscs()
1107 		rcu_assign_pointer(dev->qdisc, qdisc);  in attach_default_qdiscs()
1108 		qdisc_refcount_inc(qdisc);  in attach_default_qdiscs()
1110 		qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT, NULL);  in attach_default_qdiscs()
1111 		if (qdisc) {  in attach_default_qdiscs()
1112 			rcu_assign_pointer(dev->qdisc, qdisc);  in attach_default_qdiscs()
1113 			qdisc->ops->attach(qdisc);  in attach_default_qdiscs()
1116 	qdisc = rtnl_dereference(dev->qdisc);  in attach_default_qdiscs()
1119 	if (qdisc == &noop_qdisc) {  in attach_default_qdiscs()
1125 		qdisc = txq->qdisc_sleeping;  in attach_default_qdiscs()
1126 		rcu_assign_pointer(dev->qdisc, qdisc);  in attach_default_qdiscs()
1127 		qdisc_refcount_inc(qdisc);  in attach_default_qdiscs()
1132 	if (qdisc != &noop_qdisc)  in attach_default_qdiscs()
1133 		qdisc_hash_add(qdisc, false);  in attach_default_qdiscs()
1147 	rcu_assign_pointer(dev_queue->qdisc, new_qdisc);  in transition_one_qdisc()
1163 	if (rtnl_dereference(dev->qdisc) == &noop_qdisc)  in dev_activate()
1182 static void qdisc_deactivate(struct Qdisc *qdisc)  in qdisc_deactivate()  argument
1184 	if (qdisc->flags & TCQ_F_BUILTIN)  in qdisc_deactivate()
1187 	set_bit(__QDISC_STATE_DEACTIVATED, &qdisc->state);  in qdisc_deactivate()
1195 	struct Qdisc *qdisc;  in dev_deactivate_queue()  local
1197 	qdisc = rtnl_dereference(dev_queue->qdisc);  in dev_deactivate_queue()
1198 	if (qdisc) {  in dev_deactivate_queue()
1199 		qdisc_deactivate(qdisc);  in dev_deactivate_queue()
1200 		rcu_assign_pointer(dev_queue->qdisc, qdisc_default);  in dev_deactivate_queue()
1208 	struct Qdisc *qdisc;  in dev_reset_queue()  local
1211 	qdisc = dev_queue->qdisc_sleeping;  in dev_reset_queue()
1212 	if (!qdisc)  in dev_reset_queue()
1215 	nolock = qdisc->flags & TCQ_F_NOLOCK;  in dev_reset_queue()
1218 		spin_lock_bh(&qdisc->seqlock);  in dev_reset_queue()
1219 	spin_lock_bh(qdisc_lock(qdisc));  in dev_reset_queue()
1221 	qdisc_reset(qdisc);  in dev_reset_queue()
1223 	spin_unlock_bh(qdisc_lock(qdisc));  in dev_reset_queue()
1225 		clear_bit(__QDISC_STATE_MISSED, &qdisc->state);  in dev_reset_queue()
1226 		spin_unlock_bh(&qdisc->seqlock);  in dev_reset_queue()
1317 	struct Qdisc *qdisc = dev_queue->qdisc_sleeping;  in qdisc_change_tx_queue_len()  local
1318 	const struct Qdisc_ops *ops = qdisc->ops;  in qdisc_change_tx_queue_len()
1321 		return ops->change_tx_queue_len(qdisc, dev->tx_queue_len);  in qdisc_change_tx_queue_len()
1351 	struct Qdisc *qdisc = _qdisc;  in dev_init_scheduler_queue()  local
1353 	rcu_assign_pointer(dev_queue->qdisc, qdisc);  in dev_init_scheduler_queue()
1354 	dev_queue->qdisc_sleeping = qdisc;  in dev_init_scheduler_queue()
1359 	rcu_assign_pointer(dev->qdisc, &noop_qdisc);  in dev_init_scheduler()
1372 	qdisc_put(rtnl_dereference(dev->qdisc));  in dev_shutdown()
1373 	rcu_assign_pointer(dev->qdisc, &noop_qdisc);  in dev_shutdown()
1464 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,  in mini_qdisc_pair_init()  argument
1467 	miniqp->miniq1.cpu_bstats = qdisc->cpu_bstats;  in mini_qdisc_pair_init()
1468 	miniqp->miniq1.cpu_qstats = qdisc->cpu_qstats;  in mini_qdisc_pair_init()
1469 	miniqp->miniq2.cpu_bstats = qdisc->cpu_bstats;  in mini_qdisc_pair_init()
1470 	miniqp->miniq2.cpu_qstats = qdisc->cpu_qstats;  in mini_qdisc_pair_init()