Support optimised rendering of YUV420P in OpenGL.

diff --git a/src/tools/dcpomatic_player.cc b/src/tools/dcpomatic_player.cc
index d3b998b55054b54e942e02a151940fc2eab184b2..d462c727110238fa75235e6b63ecb388c5ef2df3 100644 (file)
--- a/src/tools/dcpomatic_player.cc
+++ b/src/tools/dcpomatic_player.cc
@@ -261,7 +261,6 @@ public:
                }
                _controls->set_film(_viewer.film());
                _viewer.set_dcp_decode_reduction(Config::instance()->decode_reduction());
-               _viewer.set_optimise_for_j2k(true);
                _viewer.PlaybackPermitted.connect(bind(&DOMFrame::playback_permitted, this));
                _viewer.TooManyDropped.connect(bind(&DOMFrame::too_many_frames_dropped, this));
                _info = new PlayerInformation (_overall_panel, _viewer);
@@ -408,6 +407,16 @@ public:
                        if (dcp->video_frame_rate()) {
                                _film->set_video_frame_rate(dcp->video_frame_rate().get(), true);
                        }
+                       switch (dcp->video_encoding()) {
+                       case VideoEncoding::JPEG2000:
+                               _viewer.set_optimisation(Optimisation::JPEG2000);
+                               break;
+                       case VideoEncoding::MPEG2:
+                               _viewer.set_optimisation(Optimisation::MPEG2);
+                               break;
+                       case VideoEncoding::COUNT:
+                               DCPOMATIC_ASSERT(false);
+                       }
                } catch (ProjectFolderError &) {
                        error_dialog (
                                this,

diff --git a/src/wx/film_viewer.cc b/src/wx/film_viewer.cc
index b6f8096e85dda7dcfb07a44a6e19fafb9781fc82..ad240b957a704fc4fdffbbd18cb4565617538567 100644 (file)
--- a/src/wx/film_viewer.cc
+++ b/src/wx/film_viewer.cc
@@ -174,7 +174,7 @@ FilmViewer::set_film (shared_ptr<Film> film)
        }
 
        try {
-               _player.emplace(_film, _optimise_for_j2k ? Image::Alignment::COMPACT : Image::Alignment::PADDED);
+               _player.emplace(_film, _optimisation == Optimisation::NONE ? Image::Alignment::PADDED : Image::Alignment::COMPACT);
                _player->set_fast ();
                if (_dcp_decode_reduction) {
                        _player->set_dcp_decode_reduction (_dcp_decode_reduction);
@@ -235,9 +235,9 @@ void
 FilmViewer::create_butler()
 {
 #if wxCHECK_VERSION(3, 1, 0)
-       auto const j2k_gl_optimised = dynamic_pointer_cast<GLVideoView>(_video_view) && _optimise_for_j2k;
+       auto const opengl = dynamic_pointer_cast<GLVideoView>(_video_view);
 #else
-       auto const j2k_gl_optimised = false;
+       auto const opengl = false;
 #endif
 
        DCPOMATIC_ASSERT(_player);
@@ -249,9 +249,9 @@ FilmViewer::create_butler()
                _audio_channels,
                boost::bind(&PlayerVideo::force, AV_PIX_FMT_RGB24),
                VideoRange::FULL,
-               j2k_gl_optimised ? Image::Alignment::COMPACT : Image::Alignment::PADDED,
+               (opengl && _optimisation != Optimisation::NONE) ? Image::Alignment::COMPACT : Image::Alignment::PADDED,
                true,
-               j2k_gl_optimised,
+               opengl && _optimisation == Optimisation::JPEG2000,
                (Config::instance()->sound() && _audio.isStreamOpen()) ? Butler::Audio::ENABLED : Butler::Audio::DISABLED
                );
 
@@ -874,10 +874,11 @@ FilmViewer::image_changed (shared_ptr<PlayerVideo> pv)
 
 
 void
-FilmViewer::set_optimise_for_j2k (bool o)
+FilmViewer::set_optimisation(Optimisation o)
 {
-       _optimise_for_j2k = o;
-       _video_view->set_optimise_for_j2k (o);
+       _optimisation = o;
+       _video_view->set_optimisation(o);
+       destroy_and_maybe_create_butler();
 }
 
 

diff --git a/src/wx/film_viewer.h b/src/wx/film_viewer.h
index 63aa113d1cafbf2e43b3ec2922a665f44f75de62..392cf85a346024748f57db25300e4e205995d3e2 100644 (file)
--- a/src/wx/film_viewer.h
+++ b/src/wx/film_viewer.h
@@ -24,6 +24,7 @@
  */
 
 
+#include "optimisation.h"
 #include "video_view.h"
 #include "lib/change_signaller.h"
 #include "lib/config.h"
@@ -101,7 +102,7 @@ public:
        void set_outline_subtitles (boost::optional<dcpomatic::Rect<double>>);
        void set_eyes (Eyes e);
        void set_pad_black (bool p);
-       void set_optimise_for_j2k (bool o);
+       void set_optimisation(Optimisation o);
        void set_crop_guess (dcpomatic::Rect<float> crop);
        void unset_crop_guess ();
 
@@ -206,10 +207,7 @@ private:
 
        boost::optional<int> _dcp_decode_reduction;
 
-       /** true to assume that this viewer is only being used for JPEG2000 sources
-        *  so it can optimise accordingly.
-        */
-       bool _optimise_for_j2k = false;
+       Optimisation _optimisation = Optimisation::NONE;
 
        ClosedCaptionsDialog* _closed_captions_dialog = nullptr;
 

diff --git a/src/wx/gl_video_view.cc b/src/wx/gl_video_view.cc
index 06c9f268b5ef4abe37476e2394c82d7a43edd499..c96fd02a03a6bef7aa434b5c5baf645da10189e5 100644 (file)
--- a/src/wx/gl_video_view.cc
+++ b/src/wx/gl_video_view.cc
@@ -194,12 +194,15 @@ static constexpr char fragment_source[] =
 "\n"
 "in vec2 TexCoord;\n"
 "\n"
-"uniform sampler2D texture_sampler;\n"
+"uniform sampler2D texture_sampler_0;\n"
+"uniform sampler2D texture_sampler_1;\n"
+"uniform sampler2D texture_sampler_2;\n"
 /* type = 0: draw outline content rectangle
  * type = 1: draw crop guess rectangle
  * type = 2: draw XYZ image
  * type = 3: draw RGB image (with sRGB/Rec709 primaries)
  * type = 4: draw RGB image (converting from Rec2020 primaries)
+ * type = 5; draw YUV image (Y in texture_sampler_0, U in texture_sampler_1, V in texture_sampler_2)
  * See FragmentType enum below.
  */
 "uniform int type = 0;\n"
@@ -270,7 +273,7 @@ static constexpr char fragment_source[] =
 "                       FragColor = crop_guess_colour;\n"
 "                       break;\n"
 "              case 2:\n"
-"                      FragColor = texture_bicubic(texture_sampler, TexCoord);\n"
+"                      FragColor = texture_bicubic(texture_sampler_0, TexCoord);\n"
 "                      FragColor.x = pow(FragColor.x, IN_GAMMA) / DCI_COEFFICIENT;\n"
 "                      FragColor.y = pow(FragColor.y, IN_GAMMA) / DCI_COEFFICIENT;\n"
 "                      FragColor.z = pow(FragColor.z, IN_GAMMA) / DCI_COEFFICIENT;\n"
@@ -280,12 +283,22 @@ static constexpr char fragment_source[] =
 "                      FragColor.z = pow(FragColor.z, OUT_GAMMA);\n"
 "                      break;\n"
 "              case 3:\n"
-"                      FragColor = texture_bicubic(texture_sampler, TexCoord);\n"
+"                      FragColor = texture_bicubic(texture_sampler_0, TexCoord);\n"
 "                       break;\n"
 "              case 4:\n"
-"                      FragColor = texture_bicubic(texture_sampler, TexCoord);\n"
+"                      FragColor = texture_bicubic(texture_sampler_0, TexCoord);\n"
 "                      FragColor = rec2020_rec709_colour_conversion * FragColor;\n"
 "                      break;\n"
+"               case 5:\n"
+"                       float y = texture_bicubic(texture_sampler_0, TexCoord).x;\n"
+"                       float u = texture_bicubic(texture_sampler_1, TexCoord).x - 0.5;\n"
+"                       float v = texture_bicubic(texture_sampler_2, TexCoord).x - 0.5;\n"
+                       // From https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.709_conversion
+"                       FragColor.x = y + 1.5748 * v;\n"
+"                       FragColor.y = y - 0.1873 * u - 0.4681 * v;\n"
+"                       FragColor.z = y + 1.8556 * u;\n"
+"                       FragColor.a = 1;\n"
+"                       break;\n"
 "      }\n"
 "}\n";
 
@@ -297,6 +310,7 @@ enum class FragmentType
        XYZ_IMAGE = 2,
        REC709_IMAGE = 3,
        REC2020_IMAGE = 4,
+       YUV420P_IMAGE = 5,
 };
 
 
@@ -455,6 +469,18 @@ GLVideoView::setup_shaders ()
        glDeleteShader (fragment_shader);
 
        glUseProgram (program);
+       auto texture_0 = glGetUniformLocation(program, "texture_sampler_0");
+       check_gl_error("glGetUniformLocation");
+       glUniform1i(texture_0, 0);
+       check_gl_error("glUniform1i");
+       auto texture_1 = glGetUniformLocation(program, "texture_sampler_1");
+       check_gl_error("glGetUniformLocation");
+       glUniform1i(texture_1, 1);
+       check_gl_error("glUniform1i");
+       auto texture_2 = glGetUniformLocation(program, "texture_sampler_2");
+       check_gl_error("glGetUniformLocation");
+       glUniform1i(texture_2, 2);
+       check_gl_error("glUniform1i");
 
        _fragment_type = glGetUniformLocation (program, "type");
        check_gl_error ("glGetUniformLocation");
@@ -560,14 +586,18 @@ GLVideoView::draw ()
 
        glBindVertexArray(_vao);
        check_gl_error ("glBindVertexArray");
-       if (_optimise_for_j2k) {
+       if (_optimisation == Optimisation::MPEG2) {
+               glUniform1i(_fragment_type, static_cast<GLint>(FragmentType::YUV420P_IMAGE));
+       } else if (_optimisation == Optimisation::JPEG2000) {
                glUniform1i(_fragment_type, static_cast<GLint>(FragmentType::XYZ_IMAGE));
        } else if (_rec2020) {
                glUniform1i(_fragment_type, static_cast<GLint>(FragmentType::REC2020_IMAGE));
        } else {
                glUniform1i(_fragment_type, static_cast<GLint>(FragmentType::REC709_IMAGE));
        }
-       _video_texture->bind();
+       for (auto& texture: _video_textures) {
+               texture->bind();
+       }
        glDrawElements (GL_TRIANGLES, indices_video_texture_number, GL_UNSIGNED_INT, reinterpret_cast<void*>(indices_video_texture_offset * sizeof(int)));
        if (_have_subtitle_to_render) {
                glUniform1i(_fragment_type, static_cast<GLint>(FragmentType::REC709_IMAGE));
@@ -595,22 +625,39 @@ GLVideoView::draw ()
 void
 GLVideoView::set_image (shared_ptr<const PlayerVideo> pv)
 {
-       shared_ptr<const Image> video = _optimise_for_j2k ? pv->raw_image() : pv->image(boost::bind(&PlayerVideo::force, AV_PIX_FMT_RGB24), VideoRange::FULL, true);
+       shared_ptr<const Image> video;
+
+       switch (_optimisation) {
+       case Optimisation::JPEG2000:
+       case Optimisation::MPEG2:
+               video = pv->raw_image();
+               break;
+       case Optimisation::NONE:
+               video = pv->image(boost::bind(&PlayerVideo::force, AV_PIX_FMT_RGB24), VideoRange::FULL, true);
+               break;
+       }
 
        /* Only the player's black frames should be aligned at this stage, so this should
         * almost always have no work to do.
         */
        video = Image::ensure_alignment (video, Image::Alignment::COMPACT);
 
-       /** If _optimise_for_j2k is true we render a XYZ image, doing the colourspace
+       /** If _optimisation is J2K we render a XYZ image, doing the colourspace
         *  conversion, scaling and video range conversion in the GL shader.
+        *  Similarly for MPEG2 we do YUV -> RGB and scaling in the shader.
         *  Otherwise we render a RGB image without any shader-side processing.
         */
 
-       _video_texture->set (video);
+       if (_optimisation == Optimisation::MPEG2) {
+               for (int i = 0; i < 3; ++i) {
+                       _video_textures[i]->set(video, i);
+               }
+       } else {
+               _video_textures[0]->set(video, 0);
+       }
 
        auto const text = pv->text();
-       _have_subtitle_to_render = static_cast<bool>(text) && _optimise_for_j2k;
+       _have_subtitle_to_render = static_cast<bool>(text) && _optimisation != Optimisation::NONE;
        if (_have_subtitle_to_render) {
                /* opt: only do this if it's a new subtitle? */
                DCPOMATIC_ASSERT (text->image->alignment() == Image::Alignment::COMPACT);
@@ -707,9 +754,9 @@ GLVideoView::set_image (shared_ptr<const PlayerVideo> pv)
        auto const sizing_changed = _last_canvas_size.changed() || _last_inter_position.changed() || _last_inter_size.changed() || _last_out_size.changed();
 
        if (sizing_changed) {
-               const auto video = _optimise_for_j2k ?
-                       Rectangle(canvas_size, inter_position.x + x_offset, inter_position.y + y_offset, inter_size)
-                       : Rectangle(canvas_size, x_offset, y_offset, out_size);
+               const auto video = _optimisation == Optimisation::NONE
+                       ? Rectangle(canvas_size, x_offset, y_offset, out_size)
+                       : Rectangle(canvas_size, inter_position.x + x_offset, inter_position.y + y_offset, inter_size);
 
                glBufferSubData (GL_ARRAY_BUFFER, array_buffer_video_offset, video.size(), video.vertices());
                check_gl_error ("glBufferSubData (video)");
@@ -739,14 +786,16 @@ GLVideoView::set_image (shared_ptr<const PlayerVideo> pv)
        _rec2020 = pv->colour_conversion() && pv->colour_conversion()->about_equal(dcp::ColourConversion::rec2020_to_xyz(), 1e-6);
 
        /* opt: where should these go? */
-
-       glTexParameteri (GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
-       glTexParameteri (GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
-       check_gl_error ("glTexParameteri");
-
-       glTexParameterf (GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
-       glTexParameterf (GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
-       check_gl_error ("glTexParameterf");
+       for (auto i = 0; i < 3; ++i) {
+               _video_textures[i]->bind();
+               glTexParameteri (GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+               glTexParameteri (GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+               check_gl_error ("glTexParameteri");
+
+               glTexParameterf (GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+               glTexParameterf (GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+               check_gl_error ("glTexParameterf");
+       }
 }
 
 
@@ -856,8 +905,11 @@ try
        _vsync_enabled = true;
 #endif
 
-       _video_texture.reset(new Texture(_optimise_for_j2k ? 2 : 1));
-       _subtitle_texture.reset(new Texture(1));
+       for (int i = 0; i < 3; ++i) {
+               std::unique_ptr<Texture> texture(new Texture(_optimisation == Optimisation::JPEG2000 ? 2 : 1, i));
+               _video_textures.push_back(std::move(texture));
+       }
+       _subtitle_texture.reset(new Texture(1, 4));
 
        while (true) {
                boost::mutex::scoped_lock lm (_playing_mutex);
@@ -905,8 +957,9 @@ GLVideoView::request_one_shot ()
 }
 
 
-Texture::Texture (GLint unpack_alignment)
+Texture::Texture(GLint unpack_alignment, int unit)
        : _unpack_alignment (unpack_alignment)
+       , _unit(unit)
 {
        glGenTextures (1, &_name);
        check_gl_error ("glGenTextures");
@@ -922,13 +975,15 @@ Texture::~Texture ()
 void
 Texture::bind ()
 {
+       glActiveTexture(GL_TEXTURE0 + _unit);
+       check_gl_error("glActiveTexture");
        glBindTexture(GL_TEXTURE_2D, _name);
        check_gl_error ("glBindTexture");
 }
 
 
 void
-Texture::set (shared_ptr<const Image> image)
+Texture::set(shared_ptr<const Image> image, int component)
 {
        auto const create = !_size || image->size() != _size;
        _size = image->size();
@@ -941,8 +996,15 @@ Texture::set (shared_ptr<const Image> image)
        GLint internal_format;
        GLenum format;
        GLenum type;
+       int subsample = 1;
 
        switch (image->pixel_format()) {
+       case AV_PIX_FMT_YUV420P:
+               internal_format = GL_R8;
+               format = GL_RED;
+               type = GL_UNSIGNED_BYTE;
+               subsample = component > 0 ? 2 : 1;
+               break;
        case AV_PIX_FMT_BGRA:
                internal_format = GL_RGBA8;
                format = GL_BGRA;
@@ -970,10 +1032,10 @@ Texture::set (shared_ptr<const Image> image)
        bind ();
 
        if (create) {
-               glTexImage2D (GL_TEXTURE_2D, 0, internal_format, _size->width, _size->height, 0, format, type, image->data()[0]);
+               glTexImage2D (GL_TEXTURE_2D, 0, internal_format, _size->width / subsample, _size->height / subsample, 0, format, type, image->data()[component]);
                check_gl_error ("glTexImage2D");
        } else {
-               glTexSubImage2D (GL_TEXTURE_2D, 0, 0, 0, _size->width, _size->height, format, type, image->data()[0]);
+               glTexSubImage2D (GL_TEXTURE_2D, 0, 0, 0, _size->width / subsample, _size->height / subsample, format, type, image->data()[component]);
                check_gl_error ("glTexSubImage2D");
        }
 }

diff --git a/src/wx/gl_video_view.h b/src/wx/gl_video_view.h
index 69de7b76fdbcd4e9555113327ebe92259d2c358b..25750b64b509fefe32489ead99f709d7b054d315 100644 (file)
--- a/src/wx/gl_video_view.h
+++ b/src/wx/gl_video_view.h
@@ -51,18 +51,19 @@ LIBDCP_ENABLE_WARNINGS
 class Texture
 {
 public:
-       Texture (GLint unpack_alignment);
+       Texture(GLint unpack_alignment, int unit = 0);
        ~Texture ();
 
        Texture (Texture const&) = delete;
        Texture& operator= (Texture const&) = delete;
 
        void bind ();
-       void set (std::shared_ptr<const Image> image);
+       void set(std::shared_ptr<const Image> image, int component = 0);
 
 private:
        GLuint _name;
        GLint _unpack_alignment;
+       int _unit;
        boost::optional<dcp::Size> _size;
 };
 
@@ -137,7 +138,7 @@ private:
 
        boost::atomic<wxSize> _canvas_size;
        boost::atomic<bool> _rec2020;
-       std::unique_ptr<Texture> _video_texture;
+       std::vector<std::unique_ptr<Texture>> _video_textures;
        std::unique_ptr<Texture> _subtitle_texture;
        bool _have_subtitle_to_render = false;
        bool _vsync_enabled;

diff --git a/src/wx/optimisation.h b/src/wx/optimisation.h
new file mode 100644 (file)
index 0000000..5d5a019
--- /dev/null
+++ b/src/wx/optimisation.h
@@ -0,0 +1,37 @@
+/*
+    Copyright (C) 2019-2021 Carl Hetherington <cth@carlh.net>
+
+    This file is part of DCP-o-matic.
+
+    DCP-o-matic is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    DCP-o-matic is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+
+#ifndef DCPOMATIC_OPTIMISATION_H
+#define DCPOMATIC_OPTIMISATION_H
+
+
+/** An optimisation that can be requested of the FilmViewer if it is known that
+ *  all content will be the same type.
+ */
+enum class Optimisation
+{
+       JPEG2000, ///< all content is JPEG2000
+       MPEG2,    ///< all content is Interop MPEG2
+       NONE,     ///< viewer must be prepared for anything
+};
+
+
+#endif

diff --git a/src/wx/video_view.h b/src/wx/video_view.h
index 387cca9f6a0a86dda8804fe3e2ad4c92e6d6baf5..3ea03a5fd9d075e0914012995d4cbe3bed6c2b93 100644 (file)
--- a/src/wx/video_view.h
+++ b/src/wx/video_view.h
@@ -23,6 +23,7 @@
 #define DCPOMATIC_VIDEO_VIEW_H
 
 
+#include "optimisation.h"
 #include "lib/dcpomatic_time.h"
 #include "lib/exception_store.h"
 #include "lib/signaller.h"
@@ -131,8 +132,8 @@ public:
                _three_d = t;
        }
 
-       void set_optimise_for_j2k (bool o) {
-               _optimise_for_j2k = o;
+       void set_optimisation(Optimisation o) {
+               _optimisation = o;
        }
 
 protected:
@@ -180,7 +181,7 @@ protected:
 
        StateTimer _state_timer;
 
-       bool _optimise_for_j2k = false;
+       Optimisation _optimisation = Optimisation::NONE;
 
 private:
        /** Mutex protecting all the state in this class */