Use uchardet to guess encoding of subtitle files and reject non-UTF-8.

diff --git a/src/lib/exceptions.h b/src/lib/exceptions.h
index 7240611ee7757b3acc760ed0acb9f4e2d571e794..6939f81a34765b68366d5b958019299e4a3f3ab8 100644 (file)
--- a/src/lib/exceptions.h
+++ b/src/lib/exceptions.h
@@ -263,4 +263,12 @@ public:
        ProgrammingError (std::string file, int line);
 };
 
+class TextEncodingError : public StringError
+{
+public:
+       TextEncodingError (std::string s)
+               : StringError (s)
+       {}
+};
+
 #endif

diff --git a/src/lib/subrip.cc b/src/lib/subrip.cc
index f1986795270c4cf2b8680cd4145035abaadf5aa8..d4adee428456823d3c62d94d596388204e6f6f16 100644 (file)
--- a/src/lib/subrip.cc
+++ b/src/lib/subrip.cc
@@ -23,10 +23,14 @@
 #include "subrip_content.h"
 #include <sub/subrip_reader.h>
 #include <sub/collect.h>
+#include <uchardet/uchardet.h>
+#include <iostream>
 
 #include "i18n.h"
 
 using std::vector;
+using std::cout;
+using std::string;
 using boost::shared_ptr;
 
 SubRip::SubRip (shared_ptr<const SubRipContent> content)
@@ -36,6 +40,25 @@ SubRip::SubRip (shared_ptr<const SubRipContent> content)
                throw OpenFileError (content->path (0));
        }
 
+       /* Guess the encoding */
+       uchardet_t det = uchardet_new ();
+       char buffer[1024];
+       while (!feof (f)) {
+               int const n = fread (buffer, 1, sizeof (buffer), f);
+               if (uchardet_handle_data (det, buffer, n)) {
+                       break;
+               }
+       }
+
+       uchardet_data_end (det);
+       string charset = uchardet_get_charset (det);
+       uchardet_delete (det);
+
+       if (charset != "UTF-8") {
+               throw TextEncodingError (_("unrecognised character set; please use files encoded in UTF-8"));
+       }
+
+       rewind (f);
        sub::SubripReader reader (f);
        _subtitles = sub::collect<vector<sub::Subtitle> > (reader.subtitles ());
 }

diff --git a/src/tools/wscript b/src/tools/wscript
index 33a631e6e2679c4d735c2400d298ab48d0b7e719..b01eee7ca1c91bac55ca411838161e2958cc1251 100644 (file)
--- a/src/tools/wscript
+++ b/src/tools/wscript
@@ -29,6 +29,7 @@ def configure(conf):
 def build(bld):
     uselib =  'BOOST_THREAD BOOST_DATETIME OPENJPEG DCP XMLSEC CXML XMLPP AVFORMAT AVFILTER AVCODEC '
     uselib += 'AVUTIL SWSCALE POSTPROC CURL BOOST_FILESYSTEM SSH ZIP CAIROMM FONTCONFIG PANGOMM SUB MAGICK SNDFILE SAMPLERATE BOOST_REGEX '
+    uselib += 'UCHARDET '
 
     if bld.env.TARGET_WINDOWS:
         uselib += 'WINSOCK2'

diff --git a/test/wscript b/test/wscript
index 1a1038e8dba8f2cacb5ec441cdfa5afd82b0946f..a92e344eb9b2ed9f1eeffec79cedc40d6a337dc4 100644 (file)
--- a/test/wscript
+++ b/test/wscript
@@ -31,7 +31,7 @@ def build(bld):
     obj = bld(features='cxx cxxprogram')
     obj.name   = 'unit-tests'
     obj.uselib =  'BOOST_TEST BOOST_THREAD BOOST_FILESYSTEM BOOST_DATETIME SNDFILE SAMPLERATE DCP OPENJPEG FONTCONFIG CAIROMM PANGOMM XMLPP '
-    obj.uselib += 'AVFORMAT AVFILTER AVCODEC AVUTIL SWSCALE POSTPROC CXML MAGICK SUB GLIB CURL SSH XMLSEC BOOST_REGEX '
+    obj.uselib += 'AVFORMAT AVFILTER AVCODEC AVUTIL SWSCALE POSTPROC CXML MAGICK SUB GLIB CURL SSH XMLSEC BOOST_REGEX UCHARDET '
     if bld.env.TARGET_WINDOWS:
         obj.uselib += 'WINSOCK2'
     obj.use    = 'libdcpomatic2'

diff --git a/wscript b/wscript
index 3e84dcac56156e4da4bbc0197c9199763e5705bd..150e9cb1678e5fb45418adff8220e0da26309baf 100644 (file)
--- a/wscript
+++ b/wscript
@@ -159,6 +159,8 @@ def configure(conf):
     else:
         conf.check_cfg(package='libcurl', args='--cflags --libs', uselib_store='CURL', mandatory=True)
 
+    # uchardet
+    conf.check_cfg(package='uchardet', args='--cflags --libs', uselib_store='UCHARDET', mandatory=True)
 
     # libsndfile
     conf.check_cfg(package='sndfile', args='--cflags --libs', uselib_store='SNDFILE', mandatory=True)