1From ca86cad12b37bffb01ed425fc1796f6e4838ce38 Mon Sep 17 00:00:00 2001 2From: Jeffy Chen <jeffy.chen@rock-chips.com> 3Date: Fri, 12 Nov 2021 11:14:37 +0800 4Subject: [PATCH 25/33] waylandsink: Support NV12_10LE40 and 5 NV12|NV12_10LE40|NV16 (AFBC) 6 7Tested on RK356x with: 8export GST_MPP_VIDEODEC_DEFAULT_ARM_AFBC=1 9gst-play-1.0 video.mp4 --videosink=waylandsink 10 11Signed-off-by: Jeffy Chen <jeffy.chen@rock-chips.com> 12--- 13 ext/wayland/gstwaylandsink.c | 98 +++++++++++++++++++++++++++++++++++- 14 ext/wayland/wldisplay.c | 19 ++++++- 15 ext/wayland/wldisplay.h | 2 + 16 ext/wayland/wllinuxdmabuf.c | 26 +++++++++- 17 ext/wayland/wlvideoformat.c | 1 + 18 ext/wayland/wlvideoformat.h | 42 ++++++++++++++++ 19 ext/wayland/wlwindow.c | 25 +++++---- 20 ext/wayland/wlwindow.h | 2 + 21 8 files changed, 203 insertions(+), 12 deletions(-) 22 23diff --git a/ext/wayland/gstwaylandsink.c b/ext/wayland/gstwaylandsink.c 24index 07bfec0..fe7e0e3 100644 25--- a/ext/wayland/gstwaylandsink.c 26+++ b/ext/wayland/gstwaylandsink.c 27@@ -78,7 +78,7 @@ GST_DEBUG_CATEGORY (gstwayland_debug); 28 #define WL_VIDEO_FORMATS \ 29 "{ BGRx, BGRA, RGBx, xBGR, xRGB, RGBA, ABGR, ARGB, RGB, BGR, " \ 30 "RGB16, BGR16, YUY2, YVYU, UYVY, AYUV, NV12, NV21, NV16, NV61, " \ 31- "YUV9, YVU9, Y41B, I420, YV12, Y42B, v308 }" 32+ "YUV9, YVU9, Y41B, I420, YV12, Y42B, v308, NV12_10LE40 }" 33 34 static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", 35 GST_PAD_SINK, 36@@ -603,6 +603,53 @@ gst_wayland_sink_set_context (GstElement * element, GstContext * context) 37 GST_ELEMENT_CLASS (parent_class)->set_context (element, context); 38 } 39 40+static GstCaps * 41+gst_wayland_sink_fixup_caps (GstWaylandSink * sink, GstCaps * caps) 42+{ 43+ GstCaps *tmp_caps = NULL; 44+ 45+ /* HACK: Allow nv12-10le40 and arm-afbc in main caps */ 46+ 47+ if (sink->display->support_nv12_10le40) { 48+ tmp_caps = gst_caps_from_string ( 49+ GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_DMABUF, 50+ "NV12_10LE40")); 51+ 52+ /* NV15(AFBC) */ 53+ if (sink->display->support_afbc) { 54+ gst_caps_ref (tmp_caps); 55+ gst_caps_append (caps, tmp_caps); 56+ 57+ gst_caps_set_simple (tmp_caps, "arm-afbc", G_TYPE_INT, 1, NULL); 58+ } 59+ 60+ gst_caps_append (caps, tmp_caps); 61+ } 62+ 63+ /* NV12|NV16 (AFBC) */ 64+ if (sink->display->support_afbc) { 65+ if (gst_wl_display_check_format_for_dmabuf (sink->display, 66+ GST_VIDEO_FORMAT_NV12)) { 67+ tmp_caps = gst_caps_from_string ( 68+ GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_DMABUF, 69+ "NV12")); 70+ gst_caps_set_simple (tmp_caps, "arm-afbc", G_TYPE_INT, 1, NULL); 71+ gst_caps_append (caps, tmp_caps); 72+ } 73+ 74+ if (gst_wl_display_check_format_for_dmabuf (sink->display, 75+ GST_VIDEO_FORMAT_NV16)) { 76+ tmp_caps = gst_caps_from_string ( 77+ GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_DMABUF, 78+ "NV16")); 79+ gst_caps_set_simple (tmp_caps, "arm-afbc", G_TYPE_INT, 1, NULL); 80+ gst_caps_append (caps, tmp_caps); 81+ } 82+ } 83+ 84+ return caps; 85+} 86+ 87 static GstCaps * 88 gst_wayland_sink_get_caps (GstBaseSink * bsink, GstCaps * filter) 89 { 90@@ -657,6 +704,8 @@ gst_wayland_sink_get_caps (GstBaseSink * bsink, GstCaps * filter) 91 gst_structure_take_value (gst_caps_get_structure (caps, 1), "format", 92 &dmabuf_list); 93 94+ caps = gst_wayland_sink_fixup_caps (sink, caps); 95+ 96 GST_DEBUG_OBJECT (sink, "display caps: %" GST_PTR_FORMAT, caps); 97 } 98 99@@ -704,6 +753,8 @@ gst_wayland_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) 100 GstWaylandSink *sink; 101 gboolean use_dmabuf; 102 GstVideoFormat format; 103+ GstStructure *s; 104+ gint value; 105 106 sink = GST_WAYLAND_SINK (bsink); 107 108@@ -713,6 +764,15 @@ gst_wayland_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) 109 if (!gst_video_info_from_caps (&sink->video_info, caps)) 110 goto invalid_format; 111 112+ /* parse AFBC from caps */ 113+ s = gst_caps_get_structure (caps, 0); 114+ if (gst_structure_get_int (s, "arm-afbc", &value)) { 115+ if (value) 116+ GST_VIDEO_INFO_SET_AFBC (&sink->video_info); 117+ else 118+ GST_VIDEO_INFO_UNSET_AFBC (&sink->video_info); 119+ } 120+ 121 format = GST_VIDEO_INFO_FORMAT (&sink->video_info); 122 sink->video_info_changed = TRUE; 123 124@@ -758,9 +818,17 @@ gst_wayland_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query) 125 GstBufferPool *pool = NULL; 126 gboolean need_pool; 127 GstAllocator *alloc; 128+ GstStructure *s; 129+ gint value; 130 131 gst_query_parse_allocation (query, &caps, &need_pool); 132 133+ s = gst_caps_get_structure (caps, 0); 134+ if (gst_structure_get_int (s, "arm-afbc", &value) && value) { 135+ GST_DEBUG_OBJECT (sink, "no allocation for AFBC"); 136+ return FALSE; 137+ } 138+ 139 if (need_pool) 140 pool = gst_wayland_create_pool (sink, caps); 141 142@@ -771,6 +839,7 @@ gst_wayland_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query) 143 alloc = gst_wl_shm_allocator_get (); 144 gst_query_add_allocation_param (query, alloc, NULL); 145 gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); 146+ gst_query_add_allocation_meta (query, GST_VIDEO_CROP_META_API_TYPE, NULL); 147 g_object_unref (alloc); 148 149 return TRUE; 150@@ -845,6 +914,7 @@ gst_wayland_sink_show_frame (GstVideoSink * vsink, GstBuffer * buffer) 151 GstWaylandSink *sink = GST_WAYLAND_SINK (vsink); 152 GstBuffer *to_render; 153 GstWlBuffer *wlbuffer; 154+ GstVideoCropMeta *crop; 155 GstVideoMeta *vmeta; 156 GstVideoFormat format; 157 GstVideoInfo old_vinfo; 158@@ -882,6 +952,23 @@ gst_wayland_sink_show_frame (GstVideoSink * vsink, GstBuffer * buffer) 159 } 160 } 161 162+ crop = gst_buffer_get_video_crop_meta (buffer); 163+ if (crop) { 164+ GstWlWindow *window = sink->window; 165+ 166+ if (window->crop_x != crop->x || window->crop_y != crop->y || 167+ window->crop_w != crop->width || window->crop_h != crop->height) { 168+ window->crop_x = crop->x; 169+ window->crop_y = crop->y; 170+ window->crop_w = crop->width; 171+ window->crop_h = crop->height; 172+ window->crop_dirty = TRUE; 173+ 174+ GST_LOG_OBJECT (sink, 175+ "crop %dx%d-%dx%d", crop->x, crop->y, crop->width, crop->height); 176+ } 177+ } 178+ 179 /* drop buffers until we get a frame callback */ 180 if (sink->redraw_pending) { 181 GST_LOG_OBJECT (sink, "buffer %p dropped (redraw pending)", buffer); 182@@ -932,6 +1019,9 @@ gst_wayland_sink_show_frame (GstVideoSink * vsink, GstBuffer * buffer) 183 &sink->video_info); 184 } 185 186+ if (!wbuf && GST_VIDEO_INFO_IS_AFBC (&sink->video_info)) 187+ goto no_afbc; 188+ 189 if (!wbuf && gst_wl_display_check_format_for_shm (sink->display, format)) { 190 if (gst_buffer_n_memory (buffer) == 1 && gst_is_fd_memory (mem)) 191 wbuf = gst_wl_shm_memory_construct_wl_buffer (mem, sink->display, 192@@ -1060,6 +1150,12 @@ no_wl_buffer: 193 ret = GST_FLOW_ERROR; 194 goto done; 195 } 196+no_afbc: 197+ { 198+ GST_ERROR_OBJECT (sink, "could not import AFBC"); 199+ ret = GST_FLOW_ERROR; 200+ goto done; 201+ } 202 activate_failed: 203 { 204 GST_ERROR_OBJECT (sink, "failed to activate bufferpool."); 205diff --git a/ext/wayland/wldisplay.c b/ext/wayland/wldisplay.c 206index e582506..fcf2853 100644 207--- a/ext/wayland/wldisplay.c 208+++ b/ext/wayland/wldisplay.c 209@@ -145,10 +145,27 @@ dmabuf_format (void *data, struct zwp_linux_dmabuf_v1 *zwp_linux_dmabuf, 210 211 if (gst_wl_dmabuf_format_to_video_format (format) != GST_VIDEO_FORMAT_UNKNOWN) 212 g_array_append_val (self->dmabuf_formats, format); 213+ 214+ if (format == DRM_FORMAT_NV15) 215+ self->support_nv12_10le40 = TRUE; 216+} 217+ 218+static void 219+dmabuf_modifier (void *data, struct zwp_linux_dmabuf_v1 *zwp_linux_dmabuf, 220+ uint32_t format, uint32_t modifier_hi, uint32_t modifier_lo) 221+{ 222+ GstWlDisplay *self = data; 223+ uint64_t modifier = ((uint64_t) modifier_hi << 32) | modifier_lo; 224+ 225+ if (modifier == DRM_AFBC_MODIFIER) 226+ self->support_afbc = TRUE; 227+ 228+ dmabuf_format (data, zwp_linux_dmabuf, format); 229 } 230 231 static const struct zwp_linux_dmabuf_v1_listener dmabuf_listener = { 232 dmabuf_format, 233+ dmabuf_modifier, 234 }; 235 236 gboolean 237@@ -235,7 +252,7 @@ registry_handle_global (void *data, struct wl_registry *registry, 238 wl_registry_bind (registry, id, &wp_viewporter_interface, 1); 239 } else if (g_strcmp0 (interface, "zwp_linux_dmabuf_v1") == 0) { 240 self->dmabuf = 241- wl_registry_bind (registry, id, &zwp_linux_dmabuf_v1_interface, 2); 242+ wl_registry_bind (registry, id, &zwp_linux_dmabuf_v1_interface, 3); 243 zwp_linux_dmabuf_v1_add_listener (self->dmabuf, &dmabuf_listener, self); 244 } 245 } 246diff --git a/ext/wayland/wldisplay.h b/ext/wayland/wldisplay.h 247index f2025a6..29c15f6 100644 248--- a/ext/wayland/wldisplay.h 249+++ b/ext/wayland/wldisplay.h 250@@ -62,6 +62,8 @@ struct _GstWlDisplay 251 struct zwp_linux_dmabuf_v1 *dmabuf; 252 GArray *shm_formats; 253 GArray *dmabuf_formats; 254+ gboolean support_afbc; 255+ gboolean support_nv12_10le40; 256 257 /* private */ 258 gboolean own_display; 259diff --git a/ext/wayland/wllinuxdmabuf.c b/ext/wayland/wllinuxdmabuf.c 260index bc1742c..de3660a 100644 261--- a/ext/wayland/wllinuxdmabuf.c 262+++ b/ext/wayland/wllinuxdmabuf.c 263@@ -44,8 +44,10 @@ gst_wl_linux_dmabuf_construct_wl_buffer (GstBuffer * buf, 264 int format; 265 guint i, width, height; 266 guint nplanes, flags = 0; 267+ gfloat stride_scale = 1.0f; 268 struct zwp_linux_buffer_params_v1 *params; 269 ConstructBufferData data; 270+ guint64 modifier = GST_VIDEO_INFO_IS_AFBC (info) ? DRM_AFBC_MODIFIER : 0; 271 272 g_return_val_if_fail (gst_wl_display_check_format_for_dmabuf (display, 273 GST_VIDEO_INFO_FORMAT (info)), NULL); 274@@ -61,6 +63,27 @@ gst_wl_linux_dmabuf_construct_wl_buffer (GstBuffer * buf, 275 G_GSSIZE_FORMAT " (%d x %d), format %s", info->size, width, height, 276 gst_wl_dmabuf_format_to_string (format)); 277 278+ if (GST_VIDEO_INFO_IS_AFBC (info)) { 279+ /* Mali uses these formats instead */ 280+ if (format == DRM_FORMAT_NV12) { 281+ format = DRM_FORMAT_YUV420_8BIT; 282+ nplanes = 1; 283+ stride_scale = 1.5; 284+ } else if (format == DRM_FORMAT_NV15) { 285+ format = DRM_FORMAT_YUV420_10BIT; 286+ nplanes = 1; 287+ stride_scale = 1.5; 288+ } else if (format == DRM_FORMAT_NV16) { 289+ format = DRM_FORMAT_YUYV; 290+ nplanes = 1; 291+ stride_scale = 2; 292+ } else { 293+ GST_ERROR_OBJECT (mem->allocator, "unsupported format for AFBC"); 294+ data.wbuf = NULL; 295+ goto out; 296+ } 297+ } 298+ 299 /* Creation and configuration of planes */ 300 params = zwp_linux_dmabuf_v1_create_params (display->dmabuf); 301 302@@ -70,11 +93,12 @@ gst_wl_linux_dmabuf_construct_wl_buffer (GstBuffer * buf, 303 304 offset = GST_VIDEO_INFO_PLANE_OFFSET (info, i); 305 stride = GST_VIDEO_INFO_PLANE_STRIDE (info, i); 306+ stride *= stride_scale; 307 if (gst_buffer_find_memory (buf, offset, 1, &mem_idx, &length, &skip)) { 308 GstMemory *m = gst_buffer_peek_memory (buf, mem_idx); 309 gint fd = gst_dmabuf_memory_get_fd (m); 310 zwp_linux_buffer_params_v1_add (params, fd, i, m->offset + skip, 311- stride, 0, 0); 312+ stride, modifier >> 32, modifier & 0xFFFFFFFF); 313 } else { 314 GST_ERROR_OBJECT (mem->allocator, "memory does not seem to contain " 315 "enough data for the specified format"); 316diff --git a/ext/wayland/wlvideoformat.c b/ext/wayland/wlvideoformat.c 317index 68cec50..11ca051 100644 318--- a/ext/wayland/wlvideoformat.c 319+++ b/ext/wayland/wlvideoformat.c 320@@ -56,6 +56,7 @@ static const wl_VideoFormat wl_formats[] = { 321 {WL_SHM_FORMAT_UYVY, DRM_FORMAT_UYVY, GST_VIDEO_FORMAT_UYVY}, 322 {WL_SHM_FORMAT_AYUV, DRM_FORMAT_AYUV, GST_VIDEO_FORMAT_AYUV}, 323 {WL_SHM_FORMAT_NV12, DRM_FORMAT_NV12, GST_VIDEO_FORMAT_NV12}, 324+ {-1, DRM_FORMAT_NV15, GST_VIDEO_FORMAT_NV12_10LE40}, 325 {WL_SHM_FORMAT_NV21, DRM_FORMAT_NV21, GST_VIDEO_FORMAT_NV21}, 326 {WL_SHM_FORMAT_NV16, DRM_FORMAT_NV16, GST_VIDEO_FORMAT_NV16}, 327 {WL_SHM_FORMAT_NV61, DRM_FORMAT_NV61, GST_VIDEO_FORMAT_NV61}, 328diff --git a/ext/wayland/wlvideoformat.h b/ext/wayland/wlvideoformat.h 329index 331f582..ddfb1e0 100644 330--- a/ext/wayland/wlvideoformat.h 331+++ b/ext/wayland/wlvideoformat.h 332@@ -30,6 +30,48 @@ 333 334 G_BEGIN_DECLS 335 336+#ifndef DRM_FORMAT_NV15 337+#define DRM_FORMAT_NV15 fourcc_code('N', 'V', '1', '5') 338+#endif 339+ 340+#ifndef DRM_FORMAT_YUV420_8BIT 341+#define DRM_FORMAT_YUV420_8BIT fourcc_code('Y', 'U', '0', '8') 342+#endif 343+ 344+#ifndef DRM_FORMAT_YUV420_10BIT 345+#define DRM_FORMAT_YUV420_10BIT fourcc_code('Y', 'U', '1', '0') 346+#endif 347+ 348+#ifndef DRM_FORMAT_MOD_VENDOR_ARM 349+#define DRM_FORMAT_MOD_VENDOR_ARM 0x08 350+#endif 351+ 352+#ifndef DRM_FORMAT_MOD_ARM_AFBC 353+#define DRM_FORMAT_MOD_ARM_AFBC(__afbc_mode) fourcc_mod_code(ARM, __afbc_mode) 354+#endif 355+ 356+#ifndef AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 357+#define AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 (1ULL) 358+#endif 359+ 360+#ifndef AFBC_FORMAT_MOD_SPARSE 361+#define AFBC_FORMAT_MOD_SPARSE (((__u64)1) << 6) 362+#endif 363+ 364+#define DRM_AFBC_MODIFIER \ 365+ (DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_SPARSE) | \ 366+ DRM_FORMAT_MOD_ARM_AFBC(AFBC_FORMAT_MOD_BLOCK_SIZE_16x16)) 367+ 368+#ifndef GST_VIDEO_FLAG_ARM_AFBC 369+#define GST_VIDEO_FLAG_ARM_AFBC (1UL << 31) 370+#define GST_VIDEO_INFO_SET_AFBC(i) \ 371+ GST_VIDEO_INFO_FLAG_SET (i, GST_VIDEO_FLAG_ARM_AFBC) 372+#define GST_VIDEO_INFO_UNSET_AFBC(i) \ 373+ GST_VIDEO_INFO_FLAG_UNSET (i, GST_VIDEO_FLAG_ARM_AFBC) 374+#define GST_VIDEO_INFO_IS_AFBC(i) \ 375+ GST_VIDEO_INFO_FLAG_IS_SET (i, GST_VIDEO_FLAG_ARM_AFBC) 376+#endif 377+ 378 enum wl_shm_format gst_video_format_to_wl_shm_format (GstVideoFormat format); 379 gint gst_video_format_to_wl_dmabuf_format (GstVideoFormat format); 380 GstVideoFormat gst_wl_shm_format_to_video_format (enum wl_shm_format wl_format); 381diff --git a/ext/wayland/wlwindow.c b/ext/wayland/wlwindow.c 382index f4e7ca9..2ba0eee 100644 383--- a/ext/wayland/wlwindow.c 384+++ b/ext/wayland/wlwindow.c 385@@ -466,6 +466,14 @@ gst_wl_window_resize_video_surface (GstWlWindow * window, gboolean commit) 386 dst.w = window->render_rectangle.w; 387 dst.h = window->render_rectangle.h; 388 389+ if (window->crop_w && window->crop_h) { 390+ src.x = window->crop_x; 391+ src.y = window->crop_y; 392+ src.w = window->crop_w; 393+ src.h = window->crop_h; 394+ } 395+ window->crop_dirty = FALSE; 396+ 397 if (window->video_viewport) { 398 if (window->fill_mode == GST_WL_WINDOW_STRETCH) { 399 res = dst; 400@@ -479,22 +487,20 @@ gst_wl_window_resize_video_surface (GstWlWindow * window, gboolean commit) 401 402 if (src_ratio < dst_ratio) { 403 int h = src.w / dst_ratio; 404- src.y = (src.h - h) / 2; 405+ src.y += (src.h - h) / 2; 406 src.h = h; 407 } else if (src_ratio > dst_ratio) { 408 int w = src.h * dst_ratio; 409- src.x = (src.w - w) / 2; 410+ src.x += (src.w - w) / 2; 411 src.w = w; 412 } 413- 414- wp_viewport_set_source (window->video_viewport, 415- wl_fixed_from_int (src.x), wl_fixed_from_int (src.y), 416- wl_fixed_from_int (src.w), wl_fixed_from_int (src.h)); 417- 418 res = dst; 419 } 420 421 wp_viewport_set_destination (window->video_viewport, res.w, res.h); 422+ wp_viewport_set_source (window->video_viewport, 423+ wl_fixed_from_int (src.x), wl_fixed_from_int (src.y), 424+ wl_fixed_from_int (src.w), wl_fixed_from_int (src.h)); 425 } else { 426 gst_video_sink_center_rect (src, dst, &res, FALSE); 427 } 428@@ -532,13 +538,14 @@ gst_wl_window_render (GstWlWindow * window, GstWlBuffer * buffer, 429 const GstVideoInfo * info) 430 { 431 if (G_UNLIKELY (info)) { 432- window->video_width = 433- gst_util_uint64_scale_int_round (info->width, info->par_n, info->par_d); 434+ window->video_width = info->width; 435 window->video_height = info->height; 436 437 wl_subsurface_set_sync (window->video_subsurface); 438 gst_wl_window_resize_video_surface (window, FALSE); 439 gst_wl_window_set_opaque (window, info); 440+ } else if (window->crop_dirty) { 441+ gst_wl_window_resize_video_surface (window, FALSE); 442 } 443 444 if (G_LIKELY (buffer)) { 445diff --git a/ext/wayland/wlwindow.h b/ext/wayland/wlwindow.h 446index 35d9d3c..672e15a 100644 447--- a/ext/wayland/wlwindow.h 448+++ b/ext/wayland/wlwindow.h 449@@ -74,6 +74,8 @@ struct _GstWlWindow 450 451 /* the size of the video in the buffers */ 452 gint video_width, video_height; 453+ gint crop_x, crop_y, crop_w, crop_h; 454+ gboolean crop_dirty; 455 456 /* when this is not set both the area_surface and the video_surface are not 457 * visible and certain steps should be skipped */ 458-- 4592.20.1 460 461