From 7245e46453a82886739a45bd78fcdf9e8401367c Mon Sep 17 00:00:00 2001 From: Carl Hetherington Date: Sat, 11 Sep 2021 18:52:05 +0200 Subject: [PATCH] When the player is used in OpenGL mode, pass unscaled XYZ data through to the shader and do colourspace conversion there. --- src/lib/butler.cc | 2 +- src/lib/butler.h | 11 +++++ src/lib/j2k_image_proxy.cc | 2 +- src/lib/player_video.cc | 12 +++++- src/lib/player_video.h | 3 +- src/tools/dcpomatic_player.cc | 1 + src/wx/film_viewer.cc | 13 ++++++ src/wx/film_viewer.h | 6 +++ src/wx/gl_video_view.cc | 80 ++++++++++++++++++++++++++--------- src/wx/gl_video_view.h | 4 +- src/wx/video_view.h | 6 +++ 11 files changed, 114 insertions(+), 26 deletions(-) diff --git a/src/lib/butler.cc b/src/lib/butler.cc index 5a8e646aa..cbd5ba15d 100644 --- a/src/lib/butler.cc +++ b/src/lib/butler.cc @@ -323,7 +323,7 @@ try /* If the weak_ptr cannot be locked the video obviously no longer requires any work */ if (video) { LOG_TIMING("start-prepare in %1", thread_id()); - video->prepare (_pixel_format, _video_range, _aligned, _fast); + video->prepare (_pixel_format, _video_range, _aligned, _fast, _prepare_only_proxy); LOG_TIMING("finish-prepare in %1", thread_id()); } } diff --git a/src/lib/butler.h b/src/lib/butler.h index a231fd099..d31442f6c 100644 --- a/src/lib/butler.h +++ b/src/lib/butler.h @@ -80,6 +80,9 @@ public: boost::optional get_closed_caption (); void disable_audio (); + void set_prepare_only_proxy (bool p) { + _prepare_only_proxy = p; + } std::pair memory_used () const; @@ -127,6 +130,14 @@ private: bool _aligned; bool _fast; + /** true to ask PlayerVideo::prepare to only prepare the ImageProxy and not also + * the final image. We want to do this when the viewer is intending to call + * PlayerVideo::raw_image() and do the things in PlayerVideo::make_imgae() itself: + * this is the case for the GLVideoView which can do scale, pixfmt conversion etc. + * in the shader. + */ + bool _prepare_only_proxy = false; + /** If we are waiting to be refilled following a seek, this is the time we were seeking to. */ diff --git a/src/lib/j2k_image_proxy.cc b/src/lib/j2k_image_proxy.cc index 144da396d..c98273ad2 100644 --- a/src/lib/j2k_image_proxy.cc +++ b/src/lib/j2k_image_proxy.cc @@ -145,7 +145,7 @@ J2KImageProxy::prepare (optional target_size) const try { /* XXX: should check that potentially trashing _data here doesn't matter */ auto decompressed = dcp::decompress_j2k (const_cast(_data->data()), _data->size(), reduce); - _image.reset (new Image (_pixel_format, decompressed->size(), true)); + _image.reset (new Image (_pixel_format, decompressed->size(), false)); int const shift = 16 - decompressed->precision (0); diff --git a/src/lib/player_video.cc b/src/lib/player_video.cc index b0e75972c..0a6ce0d99 100644 --- a/src/lib/player_video.cc +++ b/src/lib/player_video.cc @@ -121,6 +121,14 @@ PlayerVideo::image (function pixel_format, VideoR return _image; } + +shared_ptr +PlayerVideo::raw_image () const +{ + return _in->image(_inter_size).image; +} + + /** Create an image for this frame. A lock must be held on _mutex. * @param pixel_format Function which is called to decide what pixel format the output image should be; * it is passed the pixel format of the input image from the ImageProxy, and should return the desired @@ -290,11 +298,11 @@ PlayerVideo::keep_xyz_or_rgb (AVPixelFormat p) } void -PlayerVideo::prepare (function pixel_format, VideoRange video_range, bool aligned, bool fast) +PlayerVideo::prepare (function pixel_format, VideoRange video_range, bool aligned, bool fast, bool proxy_only) { _in->prepare (_inter_size); boost::mutex::scoped_lock lm (_mutex); - if (!_image) { + if (!_image && !proxy_only) { make_image (pixel_format, video_range, aligned, fast); } } diff --git a/src/lib/player_video.h b/src/lib/player_video.h index f29684832..8134c8d4e 100644 --- a/src/lib/player_video.h +++ b/src/lib/player_video.h @@ -71,8 +71,9 @@ public: void set_text (PositionImage); - void prepare (std::function pixel_format, VideoRange video_range, bool aligned, bool fast); + void prepare (std::function pixel_format, VideoRange video_range, bool aligned, bool fast, bool proxy_only); std::shared_ptr image (std::function pixel_format, VideoRange video_range, bool aligned, bool fast) const; + std::shared_ptr raw_image () const; static AVPixelFormat force (AVPixelFormat, AVPixelFormat); static AVPixelFormat keep_xyz_or_rgb (AVPixelFormat); diff --git a/src/tools/dcpomatic_player.cc b/src/tools/dcpomatic_player.cc index 23aabadd8..e409b9731 100644 --- a/src/tools/dcpomatic_player.cc +++ b/src/tools/dcpomatic_player.cc @@ -197,6 +197,7 @@ public: _controls = new StandardControls (_overall_panel, _viewer, false); } _viewer->set_dcp_decode_reduction (Config::instance()->decode_reduction ()); + _viewer->set_optimise_for_j2k (true); _viewer->PlaybackPermitted.connect (bind(&DOMFrame::playback_permitted, this)); _viewer->Started.connect (bind(&DOMFrame::playback_started, this, _1)); _viewer->Stopped.connect (bind(&DOMFrame::playback_stopped, this, _1)); diff --git a/src/wx/film_viewer.cc b/src/wx/film_viewer.cc index 77c2e85d6..749e4ceb7 100644 --- a/src/wx/film_viewer.cc +++ b/src/wx/film_viewer.cc @@ -229,6 +229,10 @@ FilmViewer::recreate_butler () _butler->disable_audio (); } + if (dynamic_pointer_cast(_video_view) && _optimise_for_j2k) { + _butler->set_prepare_only_proxy (true); + } + _closed_captions_dialog->set_butler (_butler); resume (); @@ -772,3 +776,12 @@ FilmViewer::image_changed (shared_ptr pv) { emit (boost::bind(boost::ref(ImageChanged), pv)); } + + +void +FilmViewer::set_optimise_for_j2k (bool o) +{ + _optimise_for_j2k = o; + _video_view->set_optimise_for_j2k (o); +} + diff --git a/src/wx/film_viewer.h b/src/wx/film_viewer.h index c467eedc1..5e5bb7916 100644 --- a/src/wx/film_viewer.h +++ b/src/wx/film_viewer.h @@ -98,6 +98,7 @@ public: void set_outline_subtitles (boost::optional>); void set_eyes (Eyes e); void set_pad_black (bool p); + void set_optimise_for_j2k (bool o); void slow_refresh (); @@ -187,6 +188,11 @@ private: boost::optional _dcp_decode_reduction; + /** true to assume that this viewer is only being used for JPEG2000 sources + * so it can optimise accordingly. + */ + bool _optimise_for_j2k = false; + ClosedCaptionsDialog* _closed_captions_dialog = nullptr; bool _outline_content = false; diff --git a/src/wx/gl_video_view.cc b/src/wx/gl_video_view.cc index 7c26ae616..8fcdc3c0b 100644 --- a/src/wx/gl_video_view.cc +++ b/src/wx/gl_video_view.cc @@ -189,12 +189,22 @@ static constexpr char fragment_source[] = "in vec2 TexCoord;\n" "\n" "uniform sampler2D texture_sampler;\n" -"uniform int draw_border;\n" +/* type = 0: draw border + * type = 1: draw XYZ image + * type = 2: draw RGB image + */ +"uniform int type = 0;\n" "uniform vec4 border_colour;\n" +"uniform mat4 colour_conversion;\n" "\n" "out vec4 FragColor;\n" "\n" "vec4 cubic(float x)\n" +"\n" +"#define IN_GAMMA 2.2\n" +"#define OUT_GAMMA 0.384615385\n" // 1 / 2.6 +"#define DCI_COEFFICIENT 0.91655528\n" // 48 / 53.37 +"\n" "{\n" " float x2 = x * x;\n" " float x3 = x2 * x;\n" @@ -241,10 +251,23 @@ static constexpr char fragment_source[] = "\n" "void main()\n" "{\n" -" if (draw_border == 1) {\n" -" FragColor = border_colour;\n" -" } else {\n" -" FragColor = texture_bicubic(texture_sampler, TexCoord);\n" +" switch (type) {\n" +" case 0:\n" +" FragColor = border_colour;\n" +" break;\n" +" case 1:\n" +" FragColor = texture_bicubic(texture_sampler, TexCoord);\n" +" FragColor.x = pow(FragColor.x, IN_GAMMA) / DCI_COEFFICIENT;\n" +" FragColor.y = pow(FragColor.y, IN_GAMMA) / DCI_COEFFICIENT;\n" +" FragColor.z = pow(FragColor.z, IN_GAMMA) / DCI_COEFFICIENT;\n" +" FragColor = colour_conversion * FragColor;\n" +" FragColor.x = pow(FragColor.x, OUT_GAMMA);\n" +" FragColor.y = pow(FragColor.y, OUT_GAMMA);\n" +" FragColor.z = pow(FragColor.z, OUT_GAMMA);\n" +" break;\n" +" case 2:\n" +" FragColor = texture_bicubic(texture_sampler, TexCoord);\n" +" break;\n" " }\n" "}\n"; @@ -383,10 +406,23 @@ GLVideoView::setup_shaders () glUseProgram (program); - _draw_border = glGetUniformLocation (program, "draw_border"); + _fragment_type = glGetUniformLocation (program, "type"); check_gl_error ("glGetUniformLocation"); set_border_colour (program); + auto conversion = dcp::ColourConversion::rec709_to_xyz(); + boost::numeric::ublas::matrix matrix = conversion.xyz_to_rgb (); + GLfloat gl_matrix[] = { + static_cast(matrix(0, 0)), static_cast(matrix(0, 1)), static_cast(matrix(0, 2)), 0.0f, + static_cast(matrix(1, 0)), static_cast(matrix(1, 1)), static_cast(matrix(1, 2)), 0.0f, + static_cast(matrix(2, 0)), static_cast(matrix(2, 1)), static_cast(matrix(2, 2)), 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f + }; + + auto colour_conversion = glGetUniformLocation (program, "colour_conversion"); + check_gl_error ("glGetUniformLocation"); + glUniformMatrix4fv (colour_conversion, 1, GL_TRUE, gl_matrix); + glLineWidth (2.0f); } @@ -424,10 +460,10 @@ GLVideoView::draw (Position, dcp::Size) glBindTexture(GL_TEXTURE_2D, _texture); glBindVertexArray(_vao); check_gl_error ("glBindVertexArray"); - glUniform1i(_draw_border, 0); + glUniform1i(_fragment_type, _optimise_for_j2k ? 1 : 2); glDrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0); if (_viewer->outline_content()) { - glUniform1i(_draw_border, 1); + glUniform1i(_fragment_type, 1); glDrawElements (GL_LINES, 8, GL_UNSIGNED_INT, reinterpret_cast(6 * sizeof(int))); check_gl_error ("glDrawElements"); } @@ -440,30 +476,35 @@ GLVideoView::draw (Position, dcp::Size) void -GLVideoView::set_image (shared_ptr image) +GLVideoView::set_image (shared_ptr pv) { - if (!image) { - _size = optional(); - return; - } - + auto image = _optimise_for_j2k ? pv->raw_image() : pv->image(bind(&PlayerVideo::force, _1, AV_PIX_FMT_RGB24), VideoRange::FULL, false, true); - DCPOMATIC_ASSERT (image->pixel_format() == AV_PIX_FMT_RGB24); DCPOMATIC_ASSERT (!image->aligned()); + /** If _optimise_for_j2k is true we render a XYZ image, doing the colourspace + * conversion, scaling and video range conversion in the GL shader. + * Othewise we render a RGB image without any shader-side processing. + */ + + /* XXX: video range conversion */ + /* XXX: subs */ + if (image->size() != _size) { _have_storage = false; } _size = image->size (); - glPixelStorei (GL_UNPACK_ALIGNMENT, 1); + glPixelStorei (GL_UNPACK_ALIGNMENT, _optimise_for_j2k ? 2 : 1); check_gl_error ("glPixelStorei"); + auto const format = _optimise_for_j2k ? GL_UNSIGNED_SHORT : GL_UNSIGNED_BYTE; + if (_have_storage) { - glTexSubImage2D (GL_TEXTURE_2D, 0, 0, 0, _size->width, _size->height, GL_RGB, GL_UNSIGNED_BYTE, image->data()[0]); + glTexSubImage2D (GL_TEXTURE_2D, 0, 0, 0, _size->width, _size->height, GL_RGB, format, image->data()[0]); check_gl_error ("glTexSubImage2D"); } else { - glTexImage2D (GL_TEXTURE_2D, 0, GL_RGBA8, _size->width, _size->height, 0, GL_RGB, GL_UNSIGNED_BYTE, image->data()[0]); + glTexImage2D (GL_TEXTURE_2D, 0, _optimise_for_j2k ? GL_RGBA12 : GL_RGBA8, _size->width, _size->height, 0, GL_RGB, format, image->data()[0]); check_gl_error ("glTexImage2D"); auto const canvas_size = _canvas_size.load(); @@ -517,6 +558,7 @@ GLVideoView::set_image (shared_ptr image) check_gl_error ("glTexParameterf"); } + void GLVideoView::start () { @@ -566,7 +608,7 @@ GLVideoView::set_image_and_draw () { auto pv = player_video().first; if (pv) { - set_image (pv->image(bind(&PlayerVideo::force, _1, AV_PIX_FMT_RGB24), VideoRange::FULL, false, true)); + set_image (pv); draw (pv->inter_position(), pv->inter_size()); _viewer->image_changed (pv); } diff --git a/src/wx/gl_video_view.h b/src/wx/gl_video_view.h index bac195fb1..4bab64348 100644 --- a/src/wx/gl_video_view.h +++ b/src/wx/gl_video_view.h @@ -58,7 +58,7 @@ public: } private: - void set_image (std::shared_ptr image); + void set_image (std::shared_ptr pv); void set_image_and_draw (); void draw (Position inter_position, dcp::Size inter_size); void thread (); @@ -86,7 +86,7 @@ private: boost::atomic _one_shot; GLuint _vao; - GLint _draw_border; + GLint _fragment_type; bool _setup_shaders_done = false; std::shared_ptr _timer; diff --git a/src/wx/video_view.h b/src/wx/video_view.h index 9517a3bf4..5353f213f 100644 --- a/src/wx/video_view.h +++ b/src/wx/video_view.h @@ -127,6 +127,10 @@ public: _three_d = t; } + void set_optimise_for_j2k (bool o) { + _optimise_for_j2k = o; + } + protected: NextFrameResult get_next_frame (bool non_blocking); boost::optional time_until_next_frame () const; @@ -168,6 +172,8 @@ protected: StateTimer _state_timer; + bool _optimise_for_j2k = false; + private: /** Mutex protecting all the state in this class */ mutable boost::mutex _mutex; -- 2.30.2