rendersub: use ASS track YCbCrMatrix field.

libhb/common.c

@@ -6236,6 +6236,23 @@ int hb_rgb2yuv_bt2020(int rgb)
     return (y << 16) | (Cr << 8) | Cb;
 }
 
+hb_csp_convert_f hb_get_rgb2yuv_function(int color_matrix)
+{
+    switch (color_matrix)
+    {
+    case HB_COLR_MAT_BT470BG:
+    case HB_COLR_MAT_SMPTE170M:
+    case HB_COLR_MAT_FCC:
+        return hb_rgb2yuv;
+    case HB_COLR_MAT_BT2020_NCL:
+    case HB_COLR_MAT_BT2020_CL: //wrong
+        return hb_rgb2yuv_bt2020;
+    default: //assume 709 for the rest
+        break;
+    }
+    return hb_rgb2yuv_bt709;
+}
+
 void hb_compute_chroma_smoothing_coefficient(unsigned chroma_coeffs[2][4], int pix_fmt, int chroma_location)
 {
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);

libhb/decavsub.c

@@ -619,23 +619,7 @@ int decavsubWork( hb_avsub_context_t * ctx,
                     int xx, yy;
                     uint32_t argb, ayuv;
 
-                    typedef int (*csp_f)(int);
+                    hb_csp_convert_f rgb2yuv_fn = hb_get_rgb2yuv_function(ctx->job->color_matrix);
-                    csp_f rgb2yuv_fn;
-
-                    switch (ctx->job->color_matrix)
-                    {
-                        case HB_COLR_MAT_BT470BG:
-                        case HB_COLR_MAT_SMPTE170M:
-                        case HB_COLR_MAT_FCC:
-                            rgb2yuv_fn = hb_rgb2yuv;
-                            break;
-                        case HB_COLR_MAT_BT2020_NCL:
-                        case HB_COLR_MAT_BT2020_CL: //wrong
-                            rgb2yuv_fn = hb_rgb2yuv_bt2020;
-                            break;
-                        default: //assume 709 for the rest
-                            rgb2yuv_fn = hb_rgb2yuv_bt709;
-                    }
 
                     //Convert the palette at once to YUV
                     for (xx = 0; xx < rect->nb_colors; xx++)

libhb/handbrake/common.h

@@ -1628,6 +1628,9 @@ int hb_rgb2yuv(int rgb);
 int hb_rgb2yuv_bt709(int rgb);
 int hb_rgb2yuv_bt2020(int rgb);
 
+typedef int (*hb_csp_convert_f)(int);
+hb_csp_convert_f hb_get_rgb2yuv_function(int color_matrix);
+
 void hb_compute_chroma_smoothing_coefficient(unsigned chroma_coeffs[2][4],
                                              int pix_fmt, int chroma_location);
 

libhb/rendersub.c

@@ -49,6 +49,7 @@ struct hb_filter_private_s
     ASS_Track         *ssa_track;
     uint8_t            script_initialized;
     hb_box_vec_t       boxes;
+    hb_csp_convert_f   rgb2yuv_fn;
 
     // SRT
     int                line;
@@ -501,7 +502,7 @@ static hb_buffer_t * compose_subsample_ass(hb_filter_private_t *pv, const ASS_Im
             x <= frame->dst_x && x + width >= frame->dst_x + frame->w &&
             y <= frame->dst_y && y + height >= frame->dst_y + frame->h)
         {
-            const int yuv = hb_rgb2yuv_bt709(frame->color >> 8);
+            const int yuv = pv->rgb2yuv_fn(frame->color >> 8);
 
             const unsigned frame_y = (yuv >> 16) & 0xff;
             const unsigned frame_v = (yuv >> 8 ) & 0xff;
@@ -806,6 +807,40 @@ static void ssa_close(hb_filter_object_t *filter)
     filter->private_data = NULL;
 }
 
+static void ssa_work_init(hb_filter_private_t *pv, const hb_data_t *sub_data)
+{
+    // NOTE: The codec extradata is expected to be in MKV format
+    // I would like to initialize this in ssa_post_init, but when we are
+    // transcoding text subtitles to SSA, the extradata does not
+    // get initialized until the decoder is initialized.  Since
+    // decoder initialization happens after filter initialization,
+    // we need to postpone this.
+    ass_process_codec_private(pv->ssa_track, (const char *)sub_data->bytes, sub_data->size);
+
+    switch(pv->ssa_track->YCbCrMatrix)
+    {
+    case YCBCR_DEFAULT: //No YCbCrMatrix header: VSFilter default
+    case YCBCR_FCC_TV:  //FCC is almost the same as 601
+    case YCBCR_FCC_PC:
+    case YCBCR_BT601_TV:
+    case YCBCR_BT601_PC:
+        pv->rgb2yuv_fn = hb_rgb2yuv;
+        break;
+    case YCBCR_BT709_TV:
+    case YCBCR_BT709_PC:
+    case YCBCR_SMPTE240M_TV:
+    case YCBCR_SMPTE240M_PC:  //240M is almost the same as 709
+        pv->rgb2yuv_fn = hb_rgb2yuv_bt709;
+        break;
+    //use video csp
+    case YCBCR_UNKNOWN://cannot parse
+    case YCBCR_NONE:   //explicitely requested no override
+    default:
+        pv->rgb2yuv_fn = hb_get_rgb2yuv_function(pv->input.color_matrix);
+        break;
+    }
+}
+
 static int ssa_work(hb_filter_object_t *filter,
                     hb_buffer_t **buf_in,
                     hb_buffer_t **buf_out)
@@ -816,15 +851,7 @@ static int ssa_work(hb_filter_object_t *filter,
 
     if (!pv->script_initialized)
     {
-        // NOTE: The codec extradata is expected to be in MKV format
+        ssa_work_init(pv, filter->subtitle->extradata);
-        // I would like to initialize this in ssa_post_init, but when we are
-        // transcoding text subtitles to SSA, the extradata does not
-        // get initialized until the decoder is initialized.  Since
-        // decoder initialization happens after filter initialization,
-        // we need to postpone this.
-        ass_process_codec_private(pv->ssa_track,
-                                  (char *)filter->subtitle->extradata->bytes,
-                                  filter->subtitle->extradata->size);
         pv->script_initialized = 1;
     }
     if (in->s.flags & HB_BUF_FLAG_EOF)
@@ -906,9 +933,7 @@ static int textsub_work(hb_filter_object_t *filter,
 
     if (!pv->script_initialized)
     {
-        ass_process_codec_private(pv->ssa_track,
+        ssa_work_init(pv, filter->subtitle->extradata);
-                                  (char *)filter->subtitle->extradata->bytes,
-                                  filter->subtitle->extradata->size);
         pv->script_initialized = 1;
     }