72 static inline void tegra_ivc_invalidate_counter(struct tegra_ivc *ivc,  in tegra_ivc_invalidate_counter()  argument
80 static inline void tegra_ivc_flush_counter(struct tegra_ivc *ivc,  in tegra_ivc_flush_counter()  argument
88 static inline ulong tegra_ivc_frame_addr(struct tegra_ivc *ivc,  in tegra_ivc_frame_addr()  argument
92 	BUG_ON(frame >= ivc->nframes);  in tegra_ivc_frame_addr()
95 	       (ivc->frame_size * frame);  in tegra_ivc_frame_addr()
98 static inline void *tegra_ivc_frame_pointer(struct tegra_ivc *ivc,  in tegra_ivc_frame_pointer()  argument
102 	return (void *)tegra_ivc_frame_addr(ivc, ch, frame);  in tegra_ivc_frame_pointer()
105 static inline void tegra_ivc_invalidate_frame(struct tegra_ivc *ivc,  in tegra_ivc_invalidate_frame()  argument
109 	ulong base = tegra_ivc_frame_addr(ivc, h, frame);  in tegra_ivc_invalidate_frame()
110 	invalidate_dcache_range(base, base + ivc->frame_size);  in tegra_ivc_invalidate_frame()
113 static inline void tegra_ivc_flush_frame(struct tegra_ivc *ivc,  in tegra_ivc_flush_frame()  argument
117 	ulong base = tegra_ivc_frame_addr(ivc, h, frame);  in tegra_ivc_flush_frame()
118 	flush_dcache_range(base, base + ivc->frame_size);  in tegra_ivc_flush_frame()
121 static inline int tegra_ivc_channel_empty(struct tegra_ivc *ivc,  in tegra_ivc_channel_empty()  argument
142 	if (w_count - r_count > ivc->nframes)  in tegra_ivc_channel_empty()
148 static inline int tegra_ivc_channel_full(struct tegra_ivc *ivc,  in tegra_ivc_channel_full()  argument
156 	       ivc->nframes;  in tegra_ivc_channel_full()
159 static inline void tegra_ivc_advance_rx(struct tegra_ivc *ivc)  in tegra_ivc_advance_rx()  argument
161 	ACCESS_ONCE(ivc->rx_channel->r_count) =  in tegra_ivc_advance_rx()
162 			ACCESS_ONCE(ivc->rx_channel->r_count) + 1;  in tegra_ivc_advance_rx()
164 	if (ivc->r_pos == ivc->nframes - 1)  in tegra_ivc_advance_rx()
165 		ivc->r_pos = 0;  in tegra_ivc_advance_rx()
167 		ivc->r_pos++;  in tegra_ivc_advance_rx()
170 static inline void tegra_ivc_advance_tx(struct tegra_ivc *ivc)  in tegra_ivc_advance_tx()  argument
172 	ACCESS_ONCE(ivc->tx_channel->w_count) =  in tegra_ivc_advance_tx()
173 			ACCESS_ONCE(ivc->tx_channel->w_count) + 1;  in tegra_ivc_advance_tx()
175 	if (ivc->w_pos == ivc->nframes - 1)  in tegra_ivc_advance_tx()
176 		ivc->w_pos = 0;  in tegra_ivc_advance_tx()
178 		ivc->w_pos++;  in tegra_ivc_advance_tx()
181 static inline int tegra_ivc_check_read(struct tegra_ivc *ivc)  in tegra_ivc_check_read()  argument
193 	if (ivc->tx_channel->state != ivc_state_established)  in tegra_ivc_check_read()
202 	if (!tegra_ivc_channel_empty(ivc, ivc->rx_channel))  in tegra_ivc_check_read()
206 	tegra_ivc_invalidate_counter(ivc, ivc->rx_channel, offset);  in tegra_ivc_check_read()
207 	return tegra_ivc_channel_empty(ivc, ivc->rx_channel) ? -ENOMEM : 0;  in tegra_ivc_check_read()
210 static inline int tegra_ivc_check_write(struct tegra_ivc *ivc)  in tegra_ivc_check_write()  argument
214 	if (ivc->tx_channel->state != ivc_state_established)  in tegra_ivc_check_write()
217 	if (!tegra_ivc_channel_full(ivc, ivc->tx_channel))  in tegra_ivc_check_write()
221 	tegra_ivc_invalidate_counter(ivc, ivc->tx_channel, offset);  in tegra_ivc_check_write()
222 	return tegra_ivc_channel_full(ivc, ivc->tx_channel) ? -ENOMEM : 0;  in tegra_ivc_check_write()
225 static inline uint32_t tegra_ivc_channel_avail_count(struct tegra_ivc *ivc,  in tegra_ivc_channel_avail_count()  argument
237 int tegra_ivc_read_get_next_frame(struct tegra_ivc *ivc, void **frame)  in tegra_ivc_read_get_next_frame()  argument
239 	int result = tegra_ivc_check_read(ivc);  in tegra_ivc_read_get_next_frame()
249 	tegra_ivc_invalidate_frame(ivc, ivc->rx_channel, ivc->r_pos);  in tegra_ivc_read_get_next_frame()
250 	*frame = tegra_ivc_frame_pointer(ivc, ivc->rx_channel, ivc->r_pos);  in tegra_ivc_read_get_next_frame()
255 int tegra_ivc_read_advance(struct tegra_ivc *ivc)  in tegra_ivc_read_advance()  argument
265 	result = tegra_ivc_check_read(ivc);  in tegra_ivc_read_advance()
269 	tegra_ivc_advance_rx(ivc);  in tegra_ivc_read_advance()
271 	tegra_ivc_flush_counter(ivc, ivc->rx_channel, offset);  in tegra_ivc_read_advance()
279 	tegra_ivc_invalidate_counter(ivc, ivc->rx_channel, offset);  in tegra_ivc_read_advance()
281 	if (tegra_ivc_channel_avail_count(ivc, ivc->rx_channel) ==  in tegra_ivc_read_advance()
282 	    ivc->nframes - 1)  in tegra_ivc_read_advance()
283 		ivc->notify(ivc);  in tegra_ivc_read_advance()
288 int tegra_ivc_write_get_next_frame(struct tegra_ivc *ivc, void **frame)  in tegra_ivc_write_get_next_frame()  argument
290 	int result = tegra_ivc_check_write(ivc);  in tegra_ivc_write_get_next_frame()
294 	*frame = tegra_ivc_frame_pointer(ivc, ivc->tx_channel, ivc->w_pos);  in tegra_ivc_write_get_next_frame()
299 int tegra_ivc_write_advance(struct tegra_ivc *ivc)  in tegra_ivc_write_advance()  argument
304 	result = tegra_ivc_check_write(ivc);  in tegra_ivc_write_advance()
308 	tegra_ivc_flush_frame(ivc, ivc->tx_channel, ivc->w_pos);  in tegra_ivc_write_advance()
315 	tegra_ivc_advance_tx(ivc);  in tegra_ivc_write_advance()
317 	tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);  in tegra_ivc_write_advance()
325 	tegra_ivc_invalidate_counter(ivc, ivc->tx_channel, offset);  in tegra_ivc_write_advance()
327 	if (tegra_ivc_channel_avail_count(ivc, ivc->tx_channel) == 1)  in tegra_ivc_write_advance()
328 		ivc->notify(ivc);  in tegra_ivc_write_advance()
352 int tegra_ivc_channel_notified(struct tegra_ivc *ivc)  in tegra_ivc_channel_notified()  argument
359 	tegra_ivc_invalidate_counter(ivc, ivc->rx_channel, offset);  in tegra_ivc_channel_notified()
360 	peer_state = ACCESS_ONCE(ivc->rx_channel->state);  in tegra_ivc_channel_notified()
374 		ivc->tx_channel->w_count = 0;  in tegra_ivc_channel_notified()
375 		ivc->rx_channel->r_count = 0;  in tegra_ivc_channel_notified()
377 		ivc->w_pos = 0;  in tegra_ivc_channel_notified()
378 		ivc->r_pos = 0;  in tegra_ivc_channel_notified()
390 		ivc->tx_channel->state = ivc_state_ack;  in tegra_ivc_channel_notified()
392 		tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);  in tegra_ivc_channel_notified()
397 		ivc->notify(ivc);  in tegra_ivc_channel_notified()
398 	} else if (ivc->tx_channel->state == ivc_state_sync &&  in tegra_ivc_channel_notified()
411 		ivc->tx_channel->w_count = 0;  in tegra_ivc_channel_notified()
412 		ivc->rx_channel->r_count = 0;  in tegra_ivc_channel_notified()
414 		ivc->w_pos = 0;  in tegra_ivc_channel_notified()
415 		ivc->r_pos = 0;  in tegra_ivc_channel_notified()
428 		ivc->tx_channel->state = ivc_state_established;  in tegra_ivc_channel_notified()
430 		tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);  in tegra_ivc_channel_notified()
435 		ivc->notify(ivc);  in tegra_ivc_channel_notified()
436 	} else if (ivc->tx_channel->state == ivc_state_ack) {  in tegra_ivc_channel_notified()
450 		ivc->tx_channel->state = ivc_state_established;  in tegra_ivc_channel_notified()
452 		tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);  in tegra_ivc_channel_notified()
457 		ivc->notify(ivc);  in tegra_ivc_channel_notified()
467 	if (ivc->tx_channel->state != ivc_state_established)  in tegra_ivc_channel_notified()
473 void tegra_ivc_channel_reset(struct tegra_ivc *ivc)  in tegra_ivc_channel_reset()  argument
477 	ivc->tx_channel->state = ivc_state_sync;  in tegra_ivc_channel_reset()
479 	tegra_ivc_flush_counter(ivc, ivc->tx_channel, offset);  in tegra_ivc_channel_reset()
480 	ivc->notify(ivc);  in tegra_ivc_channel_reset()
531 int tegra_ivc_init(struct tegra_ivc *ivc, ulong rx_base, ulong tx_base,  in tegra_ivc_init()  argument
537 	if (!ivc)  in tegra_ivc_init()
544 	ivc->rx_channel = (struct tegra_ivc_channel_header *)rx_base;  in tegra_ivc_init()
545 	ivc->tx_channel = (struct tegra_ivc_channel_header *)tx_base;  in tegra_ivc_init()
546 	ivc->w_pos = 0;  in tegra_ivc_init()
547 	ivc->r_pos = 0;  in tegra_ivc_init()
548 	ivc->nframes = nframes;  in tegra_ivc_init()
549 	ivc->frame_size = frame_size;  in tegra_ivc_init()
550 	ivc->notify = notify;  in tegra_ivc_init()