Re-work OpenGL video view to use modern shaders / APIs.
[dcpomatic.git] / src / wx / gl_video_view.cc
1 /*
2     Copyright (C) 2018-2021 Carl Hetherington <cth@carlh.net>
3
4     This file is part of DCP-o-matic.
5
6     DCP-o-matic is free software; you can redistribute it and/or modify
7     it under the terms of the GNU General Public License as published by
8     the Free Software Foundation; either version 2 of the License, or
9     (at your option) any later version.
10
11     DCP-o-matic is distributed in the hope that it will be useful,
12     but WITHOUT ANY WARRANTY; without even the implied warranty of
13     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14     GNU General Public License for more details.
15
16     You should have received a copy of the GNU General Public License
17     along with DCP-o-matic.  If not, see <http://www.gnu.org/licenses/>.
18
19 */
20
21
22 #ifdef DCPOMATIC_WINDOWS
23 #include <GL/glew.h>
24 #endif
25
26 #include "gl_video_view.h"
27 #include "film_viewer.h"
28 #include "wx_util.h"
29 #include "lib/image.h"
30 #include "lib/dcpomatic_assert.h"
31 #include "lib/exceptions.h"
32 #include "lib/cross.h"
33 #include "lib/player_video.h"
34 #include "lib/butler.h"
35 #include <boost/bind/bind.hpp>
36 #include <iostream>
37
38 #ifdef DCPOMATIC_OSX
39 #define GL_DO_NOT_WARN_IF_MULTI_GL_VERSION_HEADERS_INCLUDED
40 #include <OpenGL/OpenGL.h>
41 #include <OpenGL/gl3.h>
42 #endif
43
44 #ifdef DCPOMATIC_LINUX
45 #include <GL/glu.h>
46 #include <GL/glext.h>
47 #endif
48
49 #ifdef DCPOMATIC_WINDOWS
50 #include <GL/glu.h>
51 #include <GL/wglext.h>
52 #endif
53
54
55 using std::cout;
56 using std::shared_ptr;
57 using std::string;
58 using boost::optional;
59 #if BOOST_VERSION >= 106100
60 using namespace boost::placeholders;
61 #endif
62
63
64 static void
65 check_gl_error (char const * last)
66 {
67         GLenum const e = glGetError ();
68         if (e != GL_NO_ERROR) {
69                 throw GLError (last, e);
70         }
71 }
72
73
74 GLVideoView::GLVideoView (FilmViewer* viewer, wxWindow *parent)
75         : VideoView (viewer)
76         , _context (nullptr)
77         , _have_storage (false)
78         , _vsync_enabled (false)
79         , _playing (false)
80         , _one_shot (false)
81 {
82         wxGLAttributes attributes;
83         /* We don't need a depth buffer, and indeed there is apparently a bug with Windows/Intel HD 630
84          * which puts green lines over the OpenGL display if you have a non-zero depth buffer size.
85          * https://community.intel.com/t5/Graphics/Request-for-details-on-Intel-HD-630-green-lines-in-OpenGL-apps/m-p/1202179
86          */
87         attributes.PlatformDefaults().MinRGBA(8, 8, 8, 8).DoubleBuffer().Depth(0).EndList();
88         _canvas = new wxGLCanvas (
89                 parent, attributes, wxID_ANY, wxDefaultPosition, wxDefaultSize, wxFULL_REPAINT_ON_RESIZE
90         );
91         _canvas->Bind (wxEVT_PAINT, boost::bind(&GLVideoView::update, this));
92         _canvas->Bind (wxEVT_SIZE, boost::bind(&GLVideoView::size_changed, this, _1));
93
94         _canvas->Bind (wxEVT_TIMER, boost::bind(&GLVideoView::check_for_butler_errors, this));
95         _timer.reset (new wxTimer(_canvas));
96         _timer->Start (2000);
97 }
98
99
100 void
101 GLVideoView::size_changed (wxSizeEvent const& ev)
102 {
103         _canvas_size = ev.GetSize ();
104         Sized ();
105 }
106
107
108
109 GLVideoView::~GLVideoView ()
110 {
111         boost::this_thread::disable_interruption dis;
112
113         try {
114                 _thread.interrupt ();
115                 _thread.join ();
116         } catch (...) {}
117
118         glDeleteTextures (1, &_texture);
119 }
120
121 void
122 GLVideoView::check_for_butler_errors ()
123 {
124         if (!_viewer->butler()) {
125                 return;
126         }
127
128         try {
129                 _viewer->butler()->rethrow ();
130         } catch (DecodeError& e) {
131                 error_dialog (get(), e.what());
132         } catch (dcp::ReadError& e) {
133                 error_dialog (get(), wxString::Format(_("Could not read DCP: %s"), std_to_wx(e.what())));
134         }
135 }
136
137
138 /** Called from the UI thread */
139 void
140 GLVideoView::update ()
141 {
142         if (!_canvas->IsShownOnScreen()) {
143                 return;
144         }
145
146         /* It appears important to do this from the GUI thread; if we do it from the GL thread
147          * on Linux we get strange failures to create the context for any version of GL higher
148          * than 3.2.
149          */
150         ensure_context ();
151
152 #ifdef DCPOMATIC_OSX
153         /* macOS gives errors if we don't do this (and therefore [NSOpenGLContext setView:]) from the main thread */
154         if (!_setup_shaders_done) {
155                 setup_shaders ();
156                 _setup_shaders_done = true;
157         }
158 #endif
159
160         if (!_thread.joinable()) {
161                 _thread = boost::thread (boost::bind(&GLVideoView::thread, this));
162         }
163
164         request_one_shot ();
165
166         rethrow ();
167 }
168
169
170 static constexpr char vertex_source[] =
171 "#version 330 core\n"
172 "\n"
173 "layout (location = 0) in vec3 in_pos;\n"
174 "layout (location = 1) in vec2 in_tex_coord;\n"
175 "\n"
176 "out vec2 TexCoord;\n"
177 "\n"
178 "void main()\n"
179 "{\n"
180 "       gl_Position = vec4(in_pos, 1.0);\n"
181 "       TexCoord = in_tex_coord;\n"
182 "}\n";
183
184
185 /* Bicubic interpolation stolen from https://stackoverflow.com/questions/13501081/efficient-bicubic-filtering-code-in-glsl */
186 static constexpr char fragment_source[] =
187 "#version 330 core\n"
188 "\n"
189 "in vec2 TexCoord;\n"
190 "\n"
191 "uniform sampler2D texture_sampler;\n"
192 "uniform int draw_border;\n"
193 "uniform vec4 border_colour;\n"
194 "\n"
195 "out vec4 FragColor;\n"
196 "\n"
197 "vec4 cubic(float x)\n"
198 "{\n"
199 "    float x2 = x * x;\n"
200 "    float x3 = x2 * x;\n"
201 "    vec4 w;\n"
202 "    w.x =     -x3 + 3 * x2 - 3 * x + 1;\n"
203 "    w.y =  3 * x3 - 6 * x2         + 4;\n"
204 "    w.z = -3 * x3 + 3 * x2 + 3 * x + 1;\n"
205 "    w.w =  x3;\n"
206 "    return w / 6.f;\n"
207 "}\n"
208 "\n"
209 "vec4 texture_bicubic(sampler2D sampler, vec2 tex_coords)\n"
210 "{\n"
211 "   vec2 tex_size = textureSize(sampler, 0);\n"
212 "   vec2 inv_tex_size = 1.0 / tex_size;\n"
213 "\n"
214 "   tex_coords = tex_coords * tex_size - 0.5;\n"
215 "\n"
216 "   vec2 fxy = fract(tex_coords);\n"
217 "   tex_coords -= fxy;\n"
218 "\n"
219 "   vec4 xcubic = cubic(fxy.x);\n"
220 "   vec4 ycubic = cubic(fxy.y);\n"
221 "\n"
222 "   vec4 c = tex_coords.xxyy + vec2 (-0.5, +1.5).xyxy;\n"
223 "\n"
224 "   vec4 s = vec4(xcubic.xz + xcubic.yw, ycubic.xz + ycubic.yw);\n"
225 "   vec4 offset = c + vec4 (xcubic.yw, ycubic.yw) / s;\n"
226 "\n"
227 "   offset *= inv_tex_size.xxyy;\n"
228 "\n"
229 "   vec4 sample0 = texture(sampler, offset.xz);\n"
230 "   vec4 sample1 = texture(sampler, offset.yz);\n"
231 "   vec4 sample2 = texture(sampler, offset.xw);\n"
232 "   vec4 sample3 = texture(sampler, offset.yw);\n"
233 "\n"
234 "   float sx = s.x / (s.x + s.y);\n"
235 "   float sy = s.z / (s.z + s.w);\n"
236 "\n"
237 "   return mix(\n"
238 "          mix(sample3, sample2, sx), mix(sample1, sample0, sx)\n"
239 "          , sy);\n"
240 "}\n"
241 "\n"
242 "void main()\n"
243 "{\n"
244 "       if (draw_border == 1) {\n"
245 "               FragColor = border_colour;\n"
246 "       } else {\n"
247 "               FragColor = texture_bicubic(texture_sampler, TexCoord);\n"
248 "       }\n"
249 "}\n";
250
251
252 void
253 GLVideoView::ensure_context ()
254 {
255         if (!_context) {
256                 wxGLContextAttrs attrs;
257                 attrs.PlatformDefaults().CoreProfile().OGLVersion(4, 1).EndList();
258                 _context = new wxGLContext (_canvas, nullptr, &attrs);
259                 if (!_context->IsOK()) {
260                         throw GLError ("Making GL context", -1);
261                 }
262         }
263 }
264
265 void
266 GLVideoView::setup_shaders ()
267 {
268         DCPOMATIC_ASSERT (_canvas);
269         DCPOMATIC_ASSERT (_context);
270         auto r = _canvas->SetCurrent (*_context);
271         DCPOMATIC_ASSERT (r);
272
273 #ifdef DCPOMATIC_WINDOWS
274         r = glewInit();
275         if (r != GLEW_OK) {
276                 throw GLError(reinterpret_cast<char const*>(glewGetErrorString(r)));
277         }
278 #endif
279
280         unsigned int indices[] = {
281                 0, 1, 3, // texture triangle #1
282                 1, 2, 3, // texture triangle #2
283                 4, 5,    // border line #1
284                 5, 6,    // border line #2
285                 6, 7,    // border line #3
286                 7, 4,    // border line #4
287         };
288
289         glGenVertexArrays(1, &_vao);
290         check_gl_error ("glGenVertexArrays");
291         GLuint vbo;
292         glGenBuffers(1, &vbo);
293         check_gl_error ("glGenBuffers");
294         GLuint ebo;
295         glGenBuffers(1, &ebo);
296         check_gl_error ("glGenBuffers");
297
298         glBindVertexArray(_vao);
299         check_gl_error ("glBindVertexArray");
300
301         glBindBuffer(GL_ARRAY_BUFFER, vbo);
302         check_gl_error ("glBindBuffer");
303
304         glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, ebo);
305         check_gl_error ("glBindBuffer");
306         glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(indices), indices, GL_STATIC_DRAW);
307         check_gl_error ("glBufferData");
308
309         /* position attribute to vertex shader (location = 0) */
310         glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 5 * sizeof(float), nullptr);
311         glEnableVertexAttribArray(0);
312         /* texture coord attribute to vertex shader (location = 1) */
313         glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 5 * sizeof(float), reinterpret_cast<void*>(3 * sizeof(float)));
314         glEnableVertexAttribArray(1);
315         check_gl_error ("glEnableVertexAttribArray");
316
317         auto compile = [](GLenum type, char const* source) -> GLuint {
318                 auto shader = glCreateShader(type);
319                 DCPOMATIC_ASSERT (shader);
320                 GLchar const * src[] = { static_cast<GLchar const *>(source) };
321                 glShaderSource(shader, 1, src, nullptr);
322                 check_gl_error ("glShaderSource");
323                 glCompileShader(shader);
324                 check_gl_error ("glCompileShader");
325                 GLint ok;
326                 glGetShaderiv(shader, GL_COMPILE_STATUS, &ok);
327                 if (!ok) {
328                         GLint log_length;
329                         glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &log_length);
330                         string log;
331                         if (log_length > 0) {
332                                 char* log_char = new char[log_length];
333                                 glGetShaderInfoLog(shader, log_length, nullptr, log_char);
334                                 log = string(log_char);
335                                 delete[] log_char;
336                         }
337                         glDeleteShader(shader);
338                         throw GLError(String::compose("Could not compile shader (%1)", log).c_str(), -1);
339                 }
340                 return shader;
341         };
342
343         auto vertex_shader = compile (GL_VERTEX_SHADER, vertex_source);
344         auto fragment_shader = compile (GL_FRAGMENT_SHADER, fragment_source);
345
346         auto program = glCreateProgram();
347         check_gl_error ("glCreateProgram");
348         glAttachShader (program, vertex_shader);
349         check_gl_error ("glAttachShader");
350         glAttachShader (program, fragment_shader);
351         check_gl_error ("glAttachShader");
352         glLinkProgram (program);
353         check_gl_error ("glLinkProgram");
354         GLint ok;
355         glGetProgramiv (program, GL_LINK_STATUS, &ok);
356         if (!ok) {
357                 GLint log_length;
358                 glGetProgramiv(program, GL_INFO_LOG_LENGTH, &log_length);
359                 string log;
360                 if (log_length > 0) {
361                         char* log_char = new char[log_length];
362                         glGetProgramInfoLog(program, log_length, nullptr, log_char);
363                         log = string(log_char);
364                         delete[] log_char;
365                 }
366                 glDeleteProgram (program);
367                 throw GLError(String::compose("Could not link shader (%1)", log).c_str(), -1);
368         }
369         glDeleteShader (vertex_shader);
370         glDeleteShader (fragment_shader);
371
372         glUseProgram (program);
373
374         _draw_border = glGetUniformLocation (program, "draw_border");
375         check_gl_error ("glGetUniformLocation");
376         set_border_colour (program);
377
378         glLineWidth (2.0f);
379 }
380
381
382 void
383 GLVideoView::set_border_colour (GLuint program)
384 {
385         auto uniform = glGetUniformLocation (program, "border_colour");
386         check_gl_error ("glGetUniformLocation");
387         auto colour = outline_content_colour ();
388         glUniform4f (uniform, colour.Red() / 255.0f, colour.Green() / 255.0f, colour.Blue() / 255.0f, 1.0f);
389         check_gl_error ("glUniform4f");
390 }
391
392
393 void
394 GLVideoView::draw (Position<int>, dcp::Size)
395 {
396         auto pad = pad_colour();
397         glClearColor(pad.Red() / 255.0, pad.Green() / 255.0, pad.Blue() / 255.0, 1.0);
398         glClear (GL_COLOR_BUFFER_BIT);
399         check_gl_error ("glClear");
400
401         auto const size = _canvas_size.load();
402         int const width = size.GetWidth();
403         int const height = size.GetHeight();
404
405         if (width < 64 || height < 0) {
406                 return;
407         }
408
409         glViewport (0, 0, width, height);
410         check_gl_error ("glViewport");
411
412         glBindTexture(GL_TEXTURE_2D, _texture);
413         glBindVertexArray(_vao);
414         check_gl_error ("glBindVertexArray");
415         glUniform1i(_draw_border, 0);
416         glDrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0);
417         if (_viewer->outline_content()) {
418                 glUniform1i(_draw_border, 1);
419                 glDrawElements (GL_LINES, 8, GL_UNSIGNED_INT, reinterpret_cast<void*>(6 * sizeof(int)));
420                 check_gl_error ("glDrawElements");
421         }
422
423         glFlush();
424         check_gl_error ("glFlush");
425
426         _canvas->SwapBuffers();
427 }
428
429
430 void
431 GLVideoView::set_image (shared_ptr<const Image> image)
432 {
433         if (!image) {
434                 _size = optional<dcp::Size>();
435                 return;
436         }
437
438
439         DCPOMATIC_ASSERT (image->pixel_format() == AV_PIX_FMT_RGB24);
440         DCPOMATIC_ASSERT (!image->aligned());
441
442         if (image->size() != _size) {
443                 _have_storage = false;
444         }
445
446         _size = image->size ();
447         glPixelStorei (GL_UNPACK_ALIGNMENT, 1);
448         check_gl_error ("glPixelStorei");
449
450         if (_have_storage) {
451                 glTexSubImage2D (GL_TEXTURE_2D, 0, 0, 0, _size->width, _size->height, GL_RGB, GL_UNSIGNED_BYTE, image->data()[0]);
452                 check_gl_error ("glTexSubImage2D");
453         } else {
454                 glTexImage2D (GL_TEXTURE_2D, 0, GL_RGBA8, _size->width, _size->height, 0, GL_RGB, GL_UNSIGNED_BYTE, image->data()[0]);
455                 check_gl_error ("glTexImage2D");
456
457                 auto const canvas_size = _canvas_size.load();
458                 int const canvas_width = canvas_size.GetWidth();
459                 int const canvas_height = canvas_size.GetHeight();
460
461                 float const image_x = float(_size->width) / canvas_width;
462                 float const image_y = float(_size->height) / canvas_height;
463
464                 auto x_pixels_to_gl = [canvas_width](int x) {
465                         return (x * 2.0f / canvas_width) - 1.0f;
466                 };
467
468                 auto y_pixels_to_gl = [canvas_height](int y) {
469                         return (y * 2.0f / canvas_height) - 1.0f;
470                 };
471
472                 auto inter_position = player_video().first->inter_position();
473                 auto inter_size = player_video().first->inter_size();
474
475                 float const border_x1 = x_pixels_to_gl (inter_position.x) + 1.0f - image_x;
476                 float const border_y1 = y_pixels_to_gl (inter_position.y) + 1.0f - image_y;
477                 float const border_x2 = x_pixels_to_gl (inter_position.x + inter_size.width) + 1.0f - image_x;
478                 float const border_y2 = y_pixels_to_gl (inter_position.y + inter_size.height) + 1.0f - image_y;
479
480                 float vertices[] = {
481                         // positions                  // texture coords
482                          image_x,   image_y,   0.0f,  1.0f, 0.0f,   // top right           (index 0)
483                          image_x,  -image_y,   0.0f,  1.0f, 1.0f,   // bottom right        (index 1)
484                         -image_x,  -image_y,   0.0f,  0.0f, 1.0f,   // bottom left         (index 2)
485                         -image_x,   image_y,   0.0f,  0.0f, 0.0f,   // top left            (index 3)
486                          border_x1, border_y1, 0.0f,  0.0f, 0.0f,   // border bottom left  (index 4)
487                          border_x1, border_y2, 0.0f,  0.0f, 0.0f,   // border top left     (index 5)
488                          border_x2, border_y2, 0.0f,  0.0f, 0.0f,   // border top right    (index 6)
489                          border_x2, border_y1, 0.0f,  0.0f, 0.0f,   // border bottom right (index 7)
490                 };
491
492                 /* Set the vertex shader's input data (GL_ARRAY_BUFFER) */
493                 glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW);
494                 check_gl_error ("glBufferData");
495
496                 _have_storage = true;
497         }
498
499         glTexParameteri (GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
500         glTexParameteri (GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
501         check_gl_error ("glTexParameteri");
502
503         glTexParameterf (GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
504         glTexParameterf (GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
505         check_gl_error ("glTexParameterf");
506 }
507
508 void
509 GLVideoView::start ()
510 {
511         VideoView::start ();
512
513         boost::mutex::scoped_lock lm (_playing_mutex);
514         _playing = true;
515         _thread_work_condition.notify_all ();
516 }
517
518 void
519 GLVideoView::stop ()
520 {
521         boost::mutex::scoped_lock lm (_playing_mutex);
522         _playing = false;
523 }
524
525
526 void
527 GLVideoView::thread_playing ()
528 {
529         if (length() != dcpomatic::DCPTime()) {
530                 auto const next = position() + one_video_frame();
531
532                 if (next >= length()) {
533                         _viewer->finished ();
534                         return;
535                 }
536
537                 get_next_frame (false);
538                 set_image_and_draw ();
539         }
540
541         while (true) {
542                 optional<int> n = time_until_next_frame();
543                 if (!n || *n > 5) {
544                         break;
545                 }
546                 get_next_frame (true);
547                 add_dropped ();
548         }
549 }
550
551
552 void
553 GLVideoView::set_image_and_draw ()
554 {
555         auto pv = player_video().first;
556         if (pv) {
557                 set_image (pv->image(bind(&PlayerVideo::force, _1, AV_PIX_FMT_RGB24), VideoRange::FULL, false, true));
558                 draw (pv->inter_position(), pv->inter_size());
559                 _viewer->image_changed (pv);
560         }
561 }
562
563
564 void
565 GLVideoView::thread ()
566 try
567 {
568         start_of_thread ("GLVideoView");
569
570 #if defined(DCPOMATIC_OSX)
571         /* Without this we see errors like
572          * ../src/osx/cocoa/glcanvas.mm(194): assert ""context"" failed in SwapBuffers(): should have current context [in thread 700006970000]
573          */
574         WXGLSetCurrentContext (_context->GetWXGLContext());
575 #else
576         if (!_setup_shaders_done) {
577                 setup_shaders ();
578                 _setup_shaders_done = true;
579         }
580 #endif
581
582 #if defined(DCPOMATIC_LINUX) && defined(DCPOMATIC_HAVE_GLX_SWAP_INTERVAL_EXT)
583         if (_canvas->IsExtensionSupported("GLX_EXT_swap_control")) {
584                 /* Enable vsync */
585                 Display* dpy = wxGetX11Display();
586                 glXSwapIntervalEXT (dpy, DefaultScreen(dpy), 1);
587                 _vsync_enabled = true;
588         }
589 #endif
590
591 #ifdef DCPOMATIC_WINDOWS
592         if (_canvas->IsExtensionSupported("WGL_EXT_swap_control")) {
593                 /* Enable vsync */
594                 PFNWGLSWAPINTERVALEXTPROC swap = (PFNWGLSWAPINTERVALEXTPROC) wglGetProcAddress("wglSwapIntervalEXT");
595                 if (swap) {
596                         swap (1);
597                         _vsync_enabled = true;
598                 }
599         }
600
601 #endif
602
603 #ifdef DCPOMATIC_OSX
604         /* Enable vsync */
605         GLint swapInterval = 1;
606         CGLSetParameter (CGLGetCurrentContext(), kCGLCPSwapInterval, &swapInterval);
607         _vsync_enabled = true;
608 #endif
609
610         glGenTextures (1, &_texture);
611         check_gl_error ("glGenTextures");
612         glBindTexture (GL_TEXTURE_2D, _texture);
613         check_gl_error ("glBindTexture");
614
615         while (true) {
616                 boost::mutex::scoped_lock lm (_playing_mutex);
617                 while (!_playing && !_one_shot) {
618                         _thread_work_condition.wait (lm);
619                 }
620                 lm.unlock ();
621
622                 if (_playing) {
623                         thread_playing ();
624                 } else if (_one_shot) {
625                         _one_shot = false;
626                         set_image_and_draw ();
627                 }
628
629                 boost::this_thread::interruption_point ();
630                 dcpomatic_sleep_milliseconds (time_until_next_frame().get_value_or(0));
631         }
632
633         /* XXX: leaks _context, but that seems preferable to deleting it here
634          * without also deleting the wxGLCanvas.
635          */
636 }
637 catch (...)
638 {
639         store_current ();
640 }
641
642
643 VideoView::NextFrameResult
644 GLVideoView::display_next_frame (bool non_blocking)
645 {
646         NextFrameResult const r = get_next_frame (non_blocking);
647         request_one_shot ();
648         return r;
649 }
650
651
652 void
653 GLVideoView::request_one_shot ()
654 {
655         boost::mutex::scoped_lock lm (_playing_mutex);
656         _one_shot = true;
657         _thread_work_condition.notify_all ();
658 }
659