23 	struct bpf_map map;  member
30 static struct bpf_cgroup_storage_map *map_to_storage(struct bpf_map *map)  in map_to_storage()  argument
32 	return container_of(map, struct bpf_cgroup_storage_map, map);  in map_to_storage()
35 static bool attach_type_isolated(const struct bpf_map *map)  in attach_type_isolated()  argument
37 	return map->key_size == sizeof(struct bpf_cgroup_storage_key);  in attach_type_isolated()
40 static int bpf_cgroup_storage_key_cmp(const struct bpf_cgroup_storage_map *map,  in bpf_cgroup_storage_key_cmp()  argument
43 	if (attach_type_isolated(&map->map)) {  in bpf_cgroup_storage_key_cmp()
68 cgroup_storage_lookup(struct bpf_cgroup_storage_map *map,  in cgroup_storage_lookup()  argument
71 	struct rb_root *root = &map->root;  in cgroup_storage_lookup()
75 		spin_lock_bh(&map->lock);  in cgroup_storage_lookup()
83 		switch (bpf_cgroup_storage_key_cmp(map, key, &storage->key)) {  in cgroup_storage_lookup()
92 				spin_unlock_bh(&map->lock);  in cgroup_storage_lookup()
98 		spin_unlock_bh(&map->lock);  in cgroup_storage_lookup()
103 static int cgroup_storage_insert(struct bpf_cgroup_storage_map *map,  in cgroup_storage_insert()  argument
106 	struct rb_root *root = &map->root;  in cgroup_storage_insert()
115 		switch (bpf_cgroup_storage_key_cmp(map, &storage->key, &this->key)) {  in cgroup_storage_insert()
135 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in cgroup_storage_lookup_elem()  local
138 	storage = cgroup_storage_lookup(map, key, false);  in cgroup_storage_lookup_elem()
145 static int cgroup_storage_update_elem(struct bpf_map *map, void *key,  in cgroup_storage_update_elem()  argument
155 		     !map_value_has_spin_lock(map)))  in cgroup_storage_update_elem()
158 	storage = cgroup_storage_lookup((struct bpf_cgroup_storage_map *)map,  in cgroup_storage_update_elem()
164 		copy_map_value_locked(map, storage->buf->data, value, false);  in cgroup_storage_update_elem()
169 			   map->value_size,  in cgroup_storage_update_elem()
171 			   map->numa_node);  in cgroup_storage_update_elem()
175 	memcpy(&new->data[0], value, map->value_size);  in cgroup_storage_update_elem()
176 	check_and_init_map_lock(map, new->data);  in cgroup_storage_update_elem()
187 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in bpf_percpu_cgroup_storage_copy()  local
193 	storage = cgroup_storage_lookup(map, key, false);  in bpf_percpu_cgroup_storage_copy()
216 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in bpf_percpu_cgroup_storage_update()  local
225 	storage = cgroup_storage_lookup(map, key, false);  in bpf_percpu_cgroup_storage_update()
250 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in cgroup_storage_get_next_key()  local
253 	spin_lock_bh(&map->lock);  in cgroup_storage_get_next_key()
255 	if (list_empty(&map->list))  in cgroup_storage_get_next_key()
259 		storage = cgroup_storage_lookup(map, key, true);  in cgroup_storage_get_next_key()
267 		storage = list_first_entry(&map->list,  in cgroup_storage_get_next_key()
271 	spin_unlock_bh(&map->lock);  in cgroup_storage_get_next_key()
273 	if (attach_type_isolated(&map->map)) {  in cgroup_storage_get_next_key()
283 	spin_unlock_bh(&map->lock);  in cgroup_storage_get_next_key()
290 	struct bpf_cgroup_storage_map *map;  in cgroup_storage_map_alloc()  local
316 	map = kmalloc_node(sizeof(struct bpf_cgroup_storage_map),  in cgroup_storage_map_alloc()
318 	if (!map) {  in cgroup_storage_map_alloc()
323 	bpf_map_charge_move(&map->map.memory, &mem);  in cgroup_storage_map_alloc()
325 	/* copy mandatory map attributes */  in cgroup_storage_map_alloc()
326 	bpf_map_init_from_attr(&map->map, attr);  in cgroup_storage_map_alloc()
328 	spin_lock_init(&map->lock);  in cgroup_storage_map_alloc()
329 	map->root = RB_ROOT;  in cgroup_storage_map_alloc()
330 	INIT_LIST_HEAD(&map->list);  in cgroup_storage_map_alloc()
332 	return &map->map;  in cgroup_storage_map_alloc()
337 	struct bpf_cgroup_storage_map *map = map_to_storage(_map);  in cgroup_storage_map_free()  local
338 	struct list_head *storages = &map->list;  in cgroup_storage_map_free()
350 	WARN_ON(!RB_EMPTY_ROOT(&map->root));  in cgroup_storage_map_free()
351 	WARN_ON(!list_empty(&map->list));  in cgroup_storage_map_free()
353 	kfree(map);  in cgroup_storage_map_free()
356 static int cgroup_storage_delete_elem(struct bpf_map *map, void *key)  in cgroup_storage_delete_elem()  argument
361 static int cgroup_storage_check_btf(const struct bpf_map *map,  in cgroup_storage_check_btf()  argument
366 	if (attach_type_isolated(map)) {  in cgroup_storage_check_btf()
419 static void cgroup_storage_seq_show_elem(struct bpf_map *map, void *key,  in cgroup_storage_seq_show_elem()  argument
422 	enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);  in cgroup_storage_seq_show_elem()
427 	storage = cgroup_storage_lookup(map_to_storage(map), key, false);  in cgroup_storage_seq_show_elem()
433 	btf_type_seq_show(map->btf, map->btf_key_type_id, key, m);  in cgroup_storage_seq_show_elem()
434 	stype = cgroup_storage_type(map);  in cgroup_storage_seq_show_elem()
437 		btf_type_seq_show(map->btf, map->btf_value_type_id,  in cgroup_storage_seq_show_elem()
444 			btf_type_seq_show(map->btf, map->btf_value_type_id,  in cgroup_storage_seq_show_elem()
480 static size_t bpf_cgroup_storage_calculate_size(struct bpf_map *map, u32 *pages)  in bpf_cgroup_storage_calculate_size()  argument
484 	if (cgroup_storage_type(map) == BPF_CGROUP_STORAGE_SHARED) {  in bpf_cgroup_storage_calculate_size()
485 		size = sizeof(struct bpf_storage_buffer) + map->value_size;  in bpf_cgroup_storage_calculate_size()
489 		size = map->value_size;  in bpf_cgroup_storage_calculate_size()
501 	struct bpf_map *map;  in bpf_cgroup_storage_alloc()  local
506 	map = prog->aux->cgroup_storage[stype];  in bpf_cgroup_storage_alloc()
507 	if (!map)  in bpf_cgroup_storage_alloc()
510 	size = bpf_cgroup_storage_calculate_size(map, &pages);  in bpf_cgroup_storage_alloc()
512 	if (bpf_map_charge_memlock(map, pages))  in bpf_cgroup_storage_alloc()
516 			       __GFP_ZERO | GFP_USER, map->numa_node);  in bpf_cgroup_storage_alloc()
523 		storage->buf = kmalloc_node(size, flags, map->numa_node);  in bpf_cgroup_storage_alloc()
526 		check_and_init_map_lock(map, storage->buf->data);  in bpf_cgroup_storage_alloc()
533 	storage->map = (struct bpf_cgroup_storage_map *)map;  in bpf_cgroup_storage_alloc()
538 	bpf_map_uncharge_memlock(map, pages);  in bpf_cgroup_storage_alloc()
564 	struct bpf_map *map;  in bpf_cgroup_storage_free()  local
570 	map = &storage->map->map;  in bpf_cgroup_storage_free()
572 	bpf_cgroup_storage_calculate_size(map, &pages);  in bpf_cgroup_storage_free()
573 	bpf_map_uncharge_memlock(map, pages);  in bpf_cgroup_storage_free()
575 	stype = cgroup_storage_type(map);  in bpf_cgroup_storage_free()
586 	struct bpf_cgroup_storage_map *map;  in bpf_cgroup_storage_link()  local
594 	map = storage->map;  in bpf_cgroup_storage_link()
596 	spin_lock_bh(&map->lock);  in bpf_cgroup_storage_link()
597 	WARN_ON(cgroup_storage_insert(map, storage));  in bpf_cgroup_storage_link()
598 	list_add(&storage->list_map, &map->list);  in bpf_cgroup_storage_link()
600 	spin_unlock_bh(&map->lock);  in bpf_cgroup_storage_link()
605 	struct bpf_cgroup_storage_map *map;  in bpf_cgroup_storage_unlink()  local
611 	map = storage->map;  in bpf_cgroup_storage_unlink()
613 	spin_lock_bh(&map->lock);  in bpf_cgroup_storage_unlink()
614 	root = &map->root;  in bpf_cgroup_storage_unlink()
619 	spin_unlock_bh(&map->lock);  in bpf_cgroup_storage_unlink()