Multi-threaded frame calculation for movie clip proxies

This commit implements multi-threaded calculation of frames
when building proxies. Both scaling and undistortion steps
are now threaded.

Frames and proxy resolution are still handled one-by-one,
saving files after every single step. So if HDD is not so
fast, this commit could have not so much benefit.

Internal changes:

- Added IMB_scaleImBuf_threaded which scales given image
  buffer in multiple threads and uses bilinear filtering.

- libmv's camera intrinsics now have SetThreads() method
  which is used to specify how many OpenMP threads to use
  for buffer distortion/undistortion.

  And yeah, this code is using OpenMP for threading.

- Reshuffled a bit libmv-capi calls and added function
  BKE_tracking_distortion_set_threads to specify number
  of threads used by intrinscis.

extern/libmv/libmv-capi.cpp

@@ -821,6 +821,13 @@ struct libmv_CameraIntrinsics *libmv_ReconstructionExtractIntrinsics(struct libm
 	return (struct libmv_CameraIntrinsics *)&libmv_Reconstruction->intrinsics;
 }
 
+struct libmv_CameraIntrinsics *libmv_CameraIntrinsicsNewEmpty(void)
+{
+	libmv::CameraIntrinsics *camera_intrinsics = new libmv::CameraIntrinsics();
+
+	return (struct libmv_CameraIntrinsics *) camera_intrinsics;
+}
+
 struct libmv_CameraIntrinsics *libmv_CameraIntrinsicsNew(libmv_cameraIntrinsicsOptions *libmv_camera_intrinsics_options)
 {
 	libmv::CameraIntrinsics *camera_intrinsics = new libmv::CameraIntrinsics();
@@ -884,6 +891,13 @@ void libmv_CameraIntrinsicsUpdate(struct libmv_CameraIntrinsics *libmv_intrinsic
 	}
 }
 
+void libmv_CameraIntrinsicsSetThreads(struct libmv_CameraIntrinsics *libmv_intrinsics, int threads)
+{
+	libmv::CameraIntrinsics *camera_intrinsics = (libmv::CameraIntrinsics *) libmv_intrinsics;
+
+	camera_intrinsics->SetThreads(threads);
+}
+
 void libmv_CameraIntrinsicsExtract(struct libmv_CameraIntrinsics *libmv_intrinsics, double *focal_length,
 			double *principal_x, double *principal_y, double *k1, double *k2, double *k3, int *width, int *height)
 {

extern/libmv/libmv-capi.h

@@ -132,6 +132,8 @@ void libmv_destroyFeatures(struct libmv_Features *libmv_features);
 /* camera intrinsics */
 struct libmv_CameraIntrinsics *libmv_ReconstructionExtractIntrinsics(struct libmv_Reconstruction *libmv_Reconstruction);
 
+struct libmv_CameraIntrinsics *libmv_CameraIntrinsicsNewEmpty(void);
+
 struct libmv_CameraIntrinsics *libmv_CameraIntrinsicsNew(libmv_cameraIntrinsicsOptions *libmv_camera_intrinsics_options);
 
 struct libmv_CameraIntrinsics *libmv_CameraIntrinsicsCopy(struct libmv_CameraIntrinsics *libmv_intrinsics);
@@ -141,6 +143,8 @@ void libmv_CameraIntrinsicsDestroy(struct libmv_CameraIntrinsics *libmv_intrinsi
 void libmv_CameraIntrinsicsUpdate(struct libmv_CameraIntrinsics *libmv_intrinsics,
                                   libmv_cameraIntrinsicsOptions *libmv_camera_intrinsics_options);
 
+void libmv_CameraIntrinsicsSetThreads(struct libmv_CameraIntrinsics *libmv_intrinsics, int threads);
+
 void libmv_CameraIntrinsicsExtract(struct libmv_CameraIntrinsics *libmv_intrinsics, double *focal_length,
 			double *principal_x, double *principal_y, double *k1, double *k2, double *k3, int *width, int *height);
 

extern/libmv/libmv/simple_pipeline/camera_intrinsics.cc

@@ -62,7 +62,8 @@ CameraIntrinsics::CameraIntrinsics()
       p1_(0),
       p2_(0),
       distort_(0),
-      undistort_(0) {}
+      undistort_(0),
+      threads_(1) {}
 
 CameraIntrinsics::CameraIntrinsics(const CameraIntrinsics &from)
     : K_(from.K_),
@@ -72,7 +73,8 @@ CameraIntrinsics::CameraIntrinsics(const CameraIntrinsics &from)
       k2_(from.k2_),
       k3_(from.k3_),
       p1_(from.p1_),
-      p2_(from.p2_)
+      p2_(from.p2_),
+      threads_(from.threads_)
 {
   distort_ = copyGrid(from.distort_);
   undistort_ = copyGrid(from.undistort_);
@@ -120,6 +122,11 @@ void CameraIntrinsics::SetTangentialDistortion(double p1, double p2) {
   FreeLookupGrid();
 }
 
+void CameraIntrinsics::SetThreads(int threads)
+{
+	threads_ = threads;
+}
+
 void CameraIntrinsics::ApplyIntrinsics(double normalized_x,
                                        double normalized_y,
                                        double *image_x,
@@ -188,6 +195,7 @@ void CameraIntrinsics::ComputeLookupGrid(Grid* grid, int width, int height, doub
   double aspx = (double)w / image_width_;
   double aspy = (double)h / image_height_;
 
+  #pragma omp parallel for schedule(dynamic) num_threads(threads_) if (threads_ > 1 && height > 100)
   for (int y = 0; y < height; y++) {
     for (int x = 0; x < width; x++) {
       double src_x = (x - 0.5 * overscan * w) / aspx, src_y = (y - 0.5 * overscan * h) / aspy;
@@ -214,7 +222,8 @@ void CameraIntrinsics::ComputeLookupGrid(Grid* grid, int width, int height, doub
 // TODO(MatthiasF): cubic B-Spline image sampling, bilinear lookup
 template<typename T,int N>
 static void Warp(const Grid* grid, const T* src, T* dst,
-                 int width, int height) {
+                 int width, int height, int threads) {
+  #pragma omp parallel for schedule(dynamic) num_threads(threads) if (threads > 1 && height > 100)
   for (int y = 0; y < height; y++) {
     for (int x = 0; x < width; x++) {
       Offset offset = grid->offset[y*width+x];
@@ -306,37 +315,37 @@ void CameraIntrinsics::CheckUndistortLookupGrid(int width, int height, double ov
 
 void CameraIntrinsics::Distort(const float* src, float* dst, int width, int height, double overscan, int channels) {
   CheckDistortLookupGrid(width, height, overscan);
-       if(channels==1) Warp<float,1>(distort_,src,dst,width,height);
+       if(channels==1) Warp<float,1>(distort_,src,dst,width,height,threads_);
-  else if(channels==2) Warp<float,2>(distort_,src,dst,width,height);
+  else if(channels==2) Warp<float,2>(distort_,src,dst,width,height,threads_);
-  else if(channels==3) Warp<float,3>(distort_,src,dst,width,height);
+  else if(channels==3) Warp<float,3>(distort_,src,dst,width,height,threads_);
-  else if(channels==4) Warp<float,4>(distort_,src,dst,width,height);
+  else if(channels==4) Warp<float,4>(distort_,src,dst,width,height,threads_);
   //else assert("channels must be between 1 and 4");
 }
 
 void CameraIntrinsics::Distort(const unsigned char* src, unsigned char* dst, int width, int height, double overscan, int channels) {
   CheckDistortLookupGrid(width, height, overscan);
-       if(channels==1) Warp<unsigned char,1>(distort_,src,dst,width,height);
+       if(channels==1) Warp<unsigned char,1>(distort_,src,dst,width,height,threads_);
-  else if(channels==2) Warp<unsigned char,2>(distort_,src,dst,width,height);
+  else if(channels==2) Warp<unsigned char,2>(distort_,src,dst,width,height,threads_);
-  else if(channels==3) Warp<unsigned char,3>(distort_,src,dst,width,height);
+  else if(channels==3) Warp<unsigned char,3>(distort_,src,dst,width,height,threads_);
-  else if(channels==4) Warp<unsigned char,4>(distort_,src,dst,width,height);
+  else if(channels==4) Warp<unsigned char,4>(distort_,src,dst,width,height,threads_);
   //else assert("channels must be between 1 and 4");
 }
 
 void CameraIntrinsics::Undistort(const float* src, float* dst, int width, int height, double overscan, int channels) {
   CheckUndistortLookupGrid(width, height, overscan);
-       if(channels==1) Warp<float,1>(undistort_,src,dst,width,height);
+       if(channels==1) Warp<float,1>(undistort_,src,dst,width,height,threads_);
-  else if(channels==2) Warp<float,2>(undistort_,src,dst,width,height);
+  else if(channels==2) Warp<float,2>(undistort_,src,dst,width,height,threads_);
-  else if(channels==3) Warp<float,3>(undistort_,src,dst,width,height);
+  else if(channels==3) Warp<float,3>(undistort_,src,dst,width,height,threads_);
-  else if(channels==4) Warp<float,4>(undistort_,src,dst,width,height);
+  else if(channels==4) Warp<float,4>(undistort_,src,dst,width,height,threads_);
   //else assert("channels must be between 1 and 4");
 }
 
 void CameraIntrinsics::Undistort(const unsigned char* src, unsigned char* dst, int width, int height, double overscan, int channels) {
   CheckUndistortLookupGrid(width, height, overscan);
-       if(channels==1) Warp<unsigned char,1>(undistort_,src,dst,width,height);
+       if(channels==1) Warp<unsigned char,1>(undistort_,src,dst,width,height,threads_);
-  else if(channels==2) Warp<unsigned char,2>(undistort_,src,dst,width,height);
+  else if(channels==2) Warp<unsigned char,2>(undistort_,src,dst,width,height,threads_);
-  else if(channels==3) Warp<unsigned char,3>(undistort_,src,dst,width,height);
+  else if(channels==3) Warp<unsigned char,3>(undistort_,src,dst,width,height,threads_);
-  else if(channels==4) Warp<unsigned char,4>(undistort_,src,dst,width,height);
+  else if(channels==4) Warp<unsigned char,4>(undistort_,src,dst,width,height,threads_);
   //else assert("channels must be between 1 and 4");
 }
 

extern/libmv/libmv/simple_pipeline/camera_intrinsics.h

@@ -68,6 +68,9 @@ class CameraIntrinsics {
 
   void SetTangentialDistortion(double p1, double p2);
 
+  /// Set number of threads using for buffer distortion/undistortion
+  void SetThreads(int threads);
+
   /*!
       Apply camera intrinsics to the normalized point to get image coordinates.
 
@@ -153,6 +156,8 @@ class CameraIntrinsics {
 
   struct Grid *distort_;
   struct Grid *undistort_;
+
+  int threads_;
 };
 
 /// A human-readable representation of the camera intrinsic parameters.

source/blender/blenkernel/BKE_tracking.h

@@ -144,6 +144,7 @@ void BKE_tracking_camera_get_reconstructed_interpolate(struct MovieTracking *tra
 struct MovieDistortion *BKE_tracking_distortion_new(void);
 void BKE_tracking_distortion_update(struct MovieDistortion *distortion, struct MovieTracking *tracking,
                                     int calibration_width, int calibration_height);
+void BKE_tracking_distortion_set_threads(struct MovieDistortion *distortion, int threads);
 struct MovieDistortion *BKE_tracking_distortion_copy(struct MovieDistortion *distortion);
 struct ImBuf *BKE_tracking_distortion_exec(struct MovieDistortion *distortion, struct MovieTracking *tracking,
                                            struct ImBuf *ibuf, int width, int height, float overscan, int undistort);

source/blender/blenkernel/intern/movieclip.c

@@ -1244,7 +1244,7 @@ static void movieclip_build_proxy_ibuf(MovieClip *clip, ImBuf *ibuf, int cfra, i
 
 	scaleibuf = IMB_dupImBuf(ibuf);
 
-	IMB_scaleImBuf(scaleibuf, (short)rectx, (short)recty);
+	IMB_scaleImBuf_threaded(scaleibuf, (short)rectx, (short)recty);
 
 	quality = clip->proxy.quality;
 	scaleibuf->ftype = JPG | quality;

source/blender/blenkernel/intern/tracking.c

@@ -1462,6 +1462,8 @@ MovieDistortion *BKE_tracking_distortion_new(void)
 
 	distortion = MEM_callocN(sizeof(MovieDistortion), "BKE_tracking_distortion_create");
 
+	distortion->intrinsics = libmv_CameraIntrinsicsNewEmpty();
+
 	return distortion;
 }
 
@@ -1474,12 +1476,7 @@ void BKE_tracking_distortion_update(MovieDistortion *distortion, MovieTracking *
 	cameraIntrinscisOptionsFromTracking(&camera_intrinsics_options, tracking,
 	                                    calibration_width, calibration_height);
 
-	if (!distortion->intrinsics) {
+	libmv_CameraIntrinsicsUpdate(distortion->intrinsics, &camera_intrinsics_options);
-		distortion->intrinsics = libmv_CameraIntrinsicsNew(&camera_intrinsics_options);
-	}
-	else {
-		libmv_CameraIntrinsicsUpdate(distortion->intrinsics, &camera_intrinsics_options);
-	}
 #else
 	(void) distortion;
 	(void) tracking;
@@ -1488,6 +1485,16 @@ void BKE_tracking_distortion_update(MovieDistortion *distortion, MovieTracking *
 #endif
 }
 
+void BKE_tracking_distortion_set_threads(MovieDistortion *distortion, int threads)
+{
+#ifdef WITH_LIBMV
+	libmv_CameraIntrinsicsSetThreads(distortion->intrinsics, threads);
+#else
+	(void) distortion;
+	(void) threads;
+#endif
+}
+
 MovieDistortion *BKE_tracking_distortion_copy(MovieDistortion *distortion)
 {
 	MovieDistortion *new_distortion;

source/blender/editors/space_clip/clip_ops.c

@@ -40,6 +40,7 @@
 #include "BLI_path_util.h"
 #include "BLI_math.h"
 #include "BLI_rect.h"
+#include "BLI_threads.h"
 
 #include "BLF_translation.h"
 
@@ -1004,8 +1005,11 @@ static void proxy_startjob(void *pjv, short *stop, short *do_update, float *prog
 		}
 	}
 
-	if (build_undistort_count)
+	if (build_undistort_count) {
+		int threads = BLI_system_thread_count();
 		distortion = BKE_tracking_distortion_new();
+		BKE_tracking_distortion_set_threads(distortion, threads);
+	}
 
 	for (cfra = sfra; cfra <= efra; cfra++) {
 		if (clip->source != MCLIP_SRC_MOVIE)

source/blender/imbuf/IMB_imbuf.h

@@ -336,6 +336,12 @@ struct ImBuf *IMB_scaleImBuf(struct ImBuf *ibuf, unsigned int newx, unsigned int
  */
 struct ImBuf *IMB_scalefastImBuf(struct ImBuf *ibuf, unsigned int newx, unsigned int newy);
 
+/**
+ *
+ * \attention Defined in scaling.c
+ */
+void IMB_scaleImBuf_threaded(struct ImBuf *ibuf, unsigned int newx, unsigned int newy);
+
 /**
  *
  * \attention Defined in writeimage.c

source/blender/imbuf/intern/scaling.c

@@ -34,6 +34,7 @@
 
 #include "BLI_utildefines.h"
 #include "BLI_math_color.h"
+#include "BLI_math_interp.h"
 #include "MEM_guardedalloc.h"
 
 #include "imbuf.h"
@@ -1604,3 +1605,113 @@ struct ImBuf *IMB_scalefastImBuf(struct ImBuf *ibuf, unsigned int newx, unsigned
 	return(ibuf);
 }
 
+/* ******** threaded scaling ******** */
+
+typedef struct ScaleTreadInitData {
+	ImBuf *ibuf;
+
+	unsigned int newx;
+	unsigned int newy;
+
+	unsigned char *byte_buffer;
+	float *float_buffer;
+} ScaleTreadInitData;
+
+typedef struct ScaleThreadData {
+	ImBuf *ibuf;
+
+	unsigned int newx;
+	unsigned int newy;
+
+	int start_line;
+	int tot_line;
+
+	unsigned char *byte_buffer;
+	float *float_buffer;
+} ScaleThreadData;
+
+static void scale_thread_init(void *data_v, int start_line, int tot_line, void *init_data_v)
+{
+	ScaleThreadData *data = (ScaleThreadData *) data_v;
+	ScaleTreadInitData *init_data = (ScaleTreadInitData *) init_data_v;
+
+	data->ibuf = init_data->ibuf;
+
+	data->newx = init_data->newx;
+	data->newy = init_data->newy;
+
+	data->start_line = start_line;
+	data->tot_line = tot_line;
+
+	data->byte_buffer = init_data->byte_buffer;
+	data->float_buffer = init_data->float_buffer;
+}
+
+static void *do_scale_thread(void *data_v)
+{
+	ScaleThreadData *data = (ScaleThreadData *) data_v;
+	ImBuf *ibuf = data->ibuf;
+	int i;
+	float factor_x = (float) ibuf->x / data->newx;
+	float factor_y = (float) ibuf->y / data->newy;
+
+	for (i = 0; i < data->tot_line; i++) {
+		int y = data->start_line + i;
+		int x;
+
+		for (x = 0; x < data->newx; x++) {
+			float u = (float) x * factor_x;
+			float v = (float) y * factor_y;
+			int offset = y * data->newx + x;
+
+			if (data->byte_buffer) {
+				unsigned char *pixel = data->byte_buffer + 4 * offset;
+				BLI_bilinear_interpolation_char((unsigned char *) ibuf->rect, pixel, ibuf->x, ibuf->y, 4, u, v);
+			}
+
+			if (data->float_buffer) {
+				float *pixel = data->float_buffer + ibuf->channels * offset;
+				BLI_bilinear_interpolation_fl(ibuf->rect_float, pixel, ibuf->x, ibuf->y, ibuf->channels, u, v);
+			}
+		}
+	}
+
+	return NULL;
+}
+
+void IMB_scaleImBuf_threaded(ImBuf *ibuf, unsigned int newx, unsigned int newy)
+{
+	ScaleTreadInitData init_data = {0};
+
+	/* prepare initialization data */
+	init_data.ibuf = ibuf;
+
+	init_data.newx = newx;
+	init_data.newy = newy;
+
+	if (ibuf->rect)
+		init_data.byte_buffer = MEM_mallocN(4 * newx * newy * sizeof(char), "threaded scale byte buffer");
+
+	if (ibuf->rect_float)
+		init_data.float_buffer = MEM_mallocN(ibuf->channels * newx * newy * sizeof(float), "threaded scale float buffer");
+
+	/* actual scaling threads */
+	IMB_processor_apply_threaded(newy, sizeof(ScaleThreadData), &init_data,
+	                             scale_thread_init, do_scale_thread);
+
+	/* alter image buffer */
+	ibuf->x = newx;
+	ibuf->y = newy;
+
+	if (ibuf->rect) {
+		imb_freerectImBuf(ibuf);
+		ibuf->mall |= IB_rect;
+		ibuf->rect = (unsigned int *) init_data.byte_buffer;
+	}
+
+	if (ibuf->rect_float) {
+		imb_freerectfloatImBuf(ibuf);
+		ibuf->mall |= IB_rectfloat;
+		ibuf->rect_float = init_data.float_buffer;
+	}
+}