372 static void cobalt_newton_step(struct cobalt_vars *vars)  in cobalt_newton_step()  argument
377 	invsqrt = vars->rec_inv_sqrt;  in cobalt_newton_step()
379 	val = (3LL << 32) - ((u64)vars->count * invsqrt2);  in cobalt_newton_step()
384 	vars->rec_inv_sqrt = val;  in cobalt_newton_step()
387 static void cobalt_invsqrt(struct cobalt_vars *vars)  in cobalt_invsqrt()  argument
389 	if (vars->count < REC_INV_SQRT_CACHE)  in cobalt_invsqrt()
390 		vars->rec_inv_sqrt = cobalt_rec_inv_sqrt_cache[vars->count];  in cobalt_invsqrt()
392 		cobalt_newton_step(vars);  in cobalt_invsqrt()
423 static void cobalt_vars_init(struct cobalt_vars *vars)  in cobalt_vars_init()  argument
425 	memset(vars, 0, sizeof(*vars));  in cobalt_vars_init()
448 static bool cobalt_queue_full(struct cobalt_vars *vars,  in cobalt_queue_full()  argument
454 	if (ktime_to_ns(ktime_sub(now, vars->blue_timer)) > p->target) {  in cobalt_queue_full()
455 		up = !vars->p_drop;  in cobalt_queue_full()
456 		vars->p_drop += p->p_inc;  in cobalt_queue_full()
457 		if (vars->p_drop < p->p_inc)  in cobalt_queue_full()
458 			vars->p_drop = ~0;  in cobalt_queue_full()
459 		vars->blue_timer = now;  in cobalt_queue_full()
461 	vars->dropping = true;  in cobalt_queue_full()
462 	vars->drop_next = now;  in cobalt_queue_full()
463 	if (!vars->count)  in cobalt_queue_full()
464 		vars->count = 1;  in cobalt_queue_full()
472 static bool cobalt_queue_empty(struct cobalt_vars *vars,  in cobalt_queue_empty()  argument
478 	if (vars->p_drop &&  in cobalt_queue_empty()
479 	    ktime_to_ns(ktime_sub(now, vars->blue_timer)) > p->target) {  in cobalt_queue_empty()
480 		if (vars->p_drop < p->p_dec)  in cobalt_queue_empty()
481 			vars->p_drop = 0;  in cobalt_queue_empty()
483 			vars->p_drop -= p->p_dec;  in cobalt_queue_empty()
484 		vars->blue_timer = now;  in cobalt_queue_empty()
485 		down = !vars->p_drop;  in cobalt_queue_empty()
487 	vars->dropping = false;  in cobalt_queue_empty()
489 	if (vars->count && ktime_to_ns(ktime_sub(now, vars->drop_next)) >= 0) {  in cobalt_queue_empty()
490 		vars->count--;  in cobalt_queue_empty()
491 		cobalt_invsqrt(vars);  in cobalt_queue_empty()
492 		vars->drop_next = cobalt_control(vars->drop_next,  in cobalt_queue_empty()
494 						 vars->rec_inv_sqrt);  in cobalt_queue_empty()
503 static bool cobalt_should_drop(struct cobalt_vars *vars,  in cobalt_should_drop()  argument
529 	schedule = ktime_sub(now, vars->drop_next);  in cobalt_should_drop()
533 	next_due = vars->count && ktime_to_ns(schedule) >= 0;  in cobalt_should_drop()
535 	vars->ecn_marked = false;  in cobalt_should_drop()
538 		if (!vars->dropping) {  in cobalt_should_drop()
539 			vars->dropping = true;  in cobalt_should_drop()
540 			vars->drop_next = cobalt_control(now,  in cobalt_should_drop()
542 							 vars->rec_inv_sqrt);  in cobalt_should_drop()
544 		if (!vars->count)  in cobalt_should_drop()
545 			vars->count = 1;  in cobalt_should_drop()
546 	} else if (vars->dropping) {  in cobalt_should_drop()
547 		vars->dropping = false;  in cobalt_should_drop()
550 	if (next_due && vars->dropping) {  in cobalt_should_drop()
552 		drop = !(vars->ecn_marked = INET_ECN_set_ce(skb));  in cobalt_should_drop()
554 		vars->count++;  in cobalt_should_drop()
555 		if (!vars->count)  in cobalt_should_drop()
556 			vars->count--;  in cobalt_should_drop()
557 		cobalt_invsqrt(vars);  in cobalt_should_drop()
558 		vars->drop_next = cobalt_control(vars->drop_next,  in cobalt_should_drop()
560 						 vars->rec_inv_sqrt);  in cobalt_should_drop()
561 		schedule = ktime_sub(now, vars->drop_next);  in cobalt_should_drop()
564 			vars->count--;  in cobalt_should_drop()
565 			cobalt_invsqrt(vars);  in cobalt_should_drop()
566 			vars->drop_next = cobalt_control(vars->drop_next,  in cobalt_should_drop()
568 							 vars->rec_inv_sqrt);  in cobalt_should_drop()
569 			schedule = ktime_sub(now, vars->drop_next);  in cobalt_should_drop()
570 			next_due = vars->count && ktime_to_ns(schedule) >= 0;  in cobalt_should_drop()
575 	if (vars->p_drop)  in cobalt_should_drop()
576 		drop |= (prandom_u32() < vars->p_drop);  in cobalt_should_drop()
579 	if (!vars->count)  in cobalt_should_drop()
580 		vars->drop_next = ktime_add_ns(now, p->interval);  in cobalt_should_drop()
582 		vars->drop_next = now;  in cobalt_should_drop()