OpenShot Library | libopenshot  0.2.0
Frame.cpp
Go to the documentation of this file.
1 /**
2  * @file
3  * @brief Source file for Frame class
4  * @author Jonathan Thomas <jonathan@openshot.org>
5  *
6  * @section LICENSE
7  *
8  * Copyright (c) 2008-2014 OpenShot Studios, LLC
9  * <http://www.openshotstudios.com/>. This file is part of
10  * OpenShot Library (libopenshot), an open-source project dedicated to
11  * delivering high quality video editing and animation solutions to the
12  * world. For more information visit <http://www.openshot.org/>.
13  *
14  * OpenShot Library (libopenshot) is free software: you can redistribute it
15  * and/or modify it under the terms of the GNU Lesser General Public License
16  * as published by the Free Software Foundation, either version 3 of the
17  * License, or (at your option) any later version.
18  *
19  * OpenShot Library (libopenshot) is distributed in the hope that it will be
20  * useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22  * GNU Lesser General Public License for more details.
23  *
24  * You should have received a copy of the GNU Lesser General Public License
25  * along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
26  */
27 
28 #include "../include/Frame.h"
29 
30 using namespace std;
31 using namespace openshot;
32 
33 // Constructor - blank frame (300x200 blank image, 48kHz audio silence)
34 Frame::Frame() : number(1), pixel_ratio(1,1), channels(2), width(1), height(1), color("#000000"),
35  channel_layout(LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false),
36  max_audio_sample(0)
37 {
38  // Init the image magic and audio buffer
39  audio = std::shared_ptr<juce::AudioSampleBuffer>(new juce::AudioSampleBuffer(channels, 0));
40 
41  // initialize the audio samples to zero (silence)
42  audio->clear();
43 };
44 
45 // Constructor - image only (48kHz audio silence)
46 Frame::Frame(int64_t number, int width, int height, string color)
47  : number(number), pixel_ratio(1,1), channels(2), width(width), height(height), color(color),
48  channel_layout(LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false),
49  max_audio_sample(0)
50 {
51  // Init the image magic and audio buffer
52  audio = std::shared_ptr<juce::AudioSampleBuffer>(new juce::AudioSampleBuffer(channels, 0));
53 
54  // initialize the audio samples to zero (silence)
55  audio->clear();
56 };
57 
58 // Constructor - audio only (300x200 blank image)
59 Frame::Frame(int64_t number, int samples, int channels) :
60  number(number), pixel_ratio(1,1), channels(channels), width(1), height(1), color("#000000"),
61  channel_layout(LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false),
62  max_audio_sample(0)
63 {
64  // Init the image magic and audio buffer
65  audio = std::shared_ptr<juce::AudioSampleBuffer>(new juce::AudioSampleBuffer(channels, samples));
66 
67  // initialize the audio samples to zero (silence)
68  audio->clear();
69 };
70 
71 // Constructor - image & audio
72 Frame::Frame(int64_t number, int width, int height, string color, int samples, int channels)
73  : number(number), pixel_ratio(1,1), channels(channels), width(width), height(height), color(color),
74  channel_layout(LAYOUT_STEREO), sample_rate(44100), qbuffer(NULL), has_audio_data(false), has_image_data(false),
75  max_audio_sample(0)
76 {
77  // Init the image magic and audio buffer
78  audio = std::shared_ptr<juce::AudioSampleBuffer>(new juce::AudioSampleBuffer(channels, samples));
79 
80  // initialize the audio samples to zero (silence)
81  audio->clear();
82 };
83 
84 
85 // Copy constructor
86 Frame::Frame ( const Frame &other )
87 {
88  // copy pointers and data
89  DeepCopy(other);
90 }
91 
92 // Assignment operator
94 {
95  // copy pointers and data
96  DeepCopy(other);
97 
98  return *this;
99 }
100 
101 // Copy data and pointers from another Frame instance
102 void Frame::DeepCopy(const Frame& other)
103 {
104  number = other.number;
105  channels = other.channels;
106  width = other.width;
107  height = other.height;
108  channel_layout = other.channel_layout;
111  sample_rate = other.sample_rate;
112  pixel_ratio = Fraction(other.pixel_ratio.num, other.pixel_ratio.den);
113  color = other.color;
114 
115  if (other.image)
116  image = std::shared_ptr<QImage>(new QImage(*(other.image)));
117  if (other.audio)
118  audio = std::shared_ptr<juce::AudioSampleBuffer>(new juce::AudioSampleBuffer(*(other.audio)));
119  if (other.wave_image)
120  wave_image = std::shared_ptr<QImage>(new QImage(*(other.wave_image)));
121 }
122 
123 // Descructor
125  // Clear all pointers
126  image.reset();
127  audio.reset();
128 }
129 
130 // Display the frame image to the screen (primarily used for debugging reasons)
132 {
133  if (!QApplication::instance()) {
134  // Only create the QApplication once
135  static int argc = 1;
136  static char* argv[1] = {NULL};
137  previewApp = std::shared_ptr<QApplication>(new QApplication(argc, argv));
138  }
139 
140  // Get preview image
141  std::shared_ptr<QImage> previewImage = GetImage();
142 
143  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
144  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
145  {
146  // Calculate correct DAR (display aspect ratio)
147  int new_width = previewImage->size().width();
148  int new_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
149 
150  // Resize to fix DAR
151  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
152  }
153 
154  // Create window
155  QWidget previewWindow;
156  previewWindow.setStyleSheet("background-color: #000000;");
157  QHBoxLayout layout;
158 
159  // Create label with current frame's image
160  QLabel previewLabel;
161  previewLabel.setPixmap(QPixmap::fromImage(*previewImage));
162  previewLabel.setMask(QPixmap::fromImage(*previewImage).mask());
163  layout.addWidget(&previewLabel);
164 
165  // Show the window
166  previewWindow.setLayout(&layout);
167  previewWindow.show();
168  previewApp->exec();
169 }
170 
171 // Get an audio waveform image
172 std::shared_ptr<QImage> Frame::GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
173 {
174  // Clear any existing waveform image
175  ClearWaveform();
176 
177  // Init a list of lines
178  QVector<QPointF> lines;
179  QVector<QPointF> labels;
180 
181  // Calculate width of an image based on the # of samples
182  int total_samples = GetAudioSamplesCount();
183  if (total_samples > 0)
184  {
185  // If samples are present...
186  int new_height = 200 * audio->getNumChannels();
187  int height_padding = 20 * (audio->getNumChannels() - 1);
188  int total_height = new_height + height_padding;
189  int total_width = 0;
190 
191  // Loop through each audio channel
192  int Y = 100;
193  for (int channel = 0; channel < audio->getNumChannels(); channel++)
194  {
195  int X = 0;
196 
197  // Get audio for this channel
198  const float *samples = audio->getReadPointer(channel);
199 
200  for (int sample = 0; sample < GetAudioSamplesCount(); sample++, X++)
201  {
202  // Sample value (scaled to -100 to 100)
203  float value = samples[sample] * 100;
204 
205  // Append a line segment for each sample
206  if (value != 0.0) {
207  // LINE
208  lines.push_back(QPointF(X,Y));
209  lines.push_back(QPointF(X,Y-value));
210  }
211  else {
212  // DOT
213  lines.push_back(QPointF(X,Y));
214  lines.push_back(QPointF(X,Y));
215  }
216  }
217 
218  // Add Channel Label Coordinate
219  labels.push_back(QPointF(5, Y - 5));
220 
221  // Increment Y
222  Y += (200 + height_padding);
223  total_width = X;
224  }
225 
226  // Create blank image
227  wave_image = std::shared_ptr<QImage>(new QImage(total_width, total_height, QImage::Format_RGBA8888));
228  wave_image->fill(QColor(0,0,0,0));
229 
230  // Load QPainter with wave_image device
231  QPainter painter(wave_image.get());
232 
233  // Set pen color
234  painter.setPen(QColor(Red, Green, Blue, Alpha));
235 
236  // Draw the waveform
237  painter.drawLines(lines);
238  painter.end();
239 
240  // Loop through the channels labels (and draw the text)
241  // TODO: Configure Fonts in Qt5 correctly, so the drawText method does not crash
242 // painter.setFont(QFont(QString("Arial"), 16, 1, false));
243 // for (int channel = 0; channel < labels.size(); channel++) {
244 // stringstream label;
245 // label << "Channel " << channel;
246 // painter.drawText(labels.at(channel), QString::fromStdString(label.str()));
247 // }
248 
249  // Resize Image (if requested)
250  if (width != total_width || height != total_height) {
251  QImage scaled_wave_image = wave_image->scaled(width, height, Qt::IgnoreAspectRatio, Qt::FastTransformation);
252  wave_image = std::shared_ptr<QImage>(new QImage(scaled_wave_image));
253  }
254  }
255  else
256  {
257  // No audio samples present
258  wave_image = std::shared_ptr<QImage>(new QImage(width, height, QImage::Format_RGBA8888));
259  wave_image->fill(QColor(QString::fromStdString("#000000")));
260  }
261 
262  // Return new image
263  return wave_image;
264 }
265 
266 // Clear the waveform image (and deallocate it's memory)
268 {
269  if (wave_image)
270  wave_image.reset();
271 }
272 
273 // Get an audio waveform image pixels
274 const unsigned char* Frame::GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
275 {
276  // Get audio wave form image
277  wave_image = GetWaveform(width, height, Red, Green, Blue, Alpha);
278 
279  // Return array of pixel packets
280  return wave_image->bits();
281 }
282 
283 // Display the wave form
285 {
286  // Get audio wave form image
287  GetWaveform(720, 480, 0, 123, 255, 255);
288 
289  if (!QApplication::instance()) {
290  // Only create the QApplication once
291  static int argc = 1;
292  static char* argv[1] = {NULL};
293  previewApp = std::shared_ptr<QApplication>(new QApplication(argc, argv));
294  }
295 
296  // Create window
297  QWidget previewWindow;
298  previewWindow.setStyleSheet("background-color: #000000;");
299  QHBoxLayout layout;
300 
301  // Create label with current frame's waveform image
302  QLabel previewLabel;
303  previewLabel.setPixmap(QPixmap::fromImage(*wave_image));
304  previewLabel.setMask(QPixmap::fromImage(*wave_image).mask());
305  layout.addWidget(&previewLabel);
306 
307  // Show the window
308  previewWindow.setLayout(&layout);
309  previewWindow.show();
310  previewApp->exec();
311 
312  // Deallocate waveform image
313  ClearWaveform();
314 }
315 
316 // Get magnitude of range of samples (if channel is -1, return average of all channels for that sample)
317 float Frame::GetAudioSample(int channel, int sample, int magnitude_range)
318 {
319  if (channel > 0) {
320  // return average magnitude for a specific channel/sample range
321  return audio->getMagnitude(channel, sample, magnitude_range);
322 
323  } else {
324  // Return average magnitude for all channels
325  return audio->getMagnitude(sample, magnitude_range);
326  }
327 }
328 
329 // Get an array of sample data
330 float* Frame::GetAudioSamples(int channel)
331 {
332  // return JUCE audio data for this channel
333  return audio->getWritePointer(channel);
334 }
335 
336 // Get a planar array of sample data, using any sample rate
337 float* Frame::GetPlanarAudioSamples(int new_sample_rate, AudioResampler* resampler, int* sample_count)
338 {
339  float *output = NULL;
340  AudioSampleBuffer *buffer(audio.get());
341  int num_of_channels = audio->getNumChannels();
342  int num_of_samples = GetAudioSamplesCount();
343 
344  // Resample to new sample rate (if needed)
345  if (new_sample_rate != sample_rate)
346  {
347  // YES, RESAMPLE AUDIO
348  resampler->SetBuffer(audio.get(), sample_rate, new_sample_rate);
349 
350  // Resample data, and return new buffer pointer
351  buffer = resampler->GetResampledBuffer();
352 
353  // Update num_of_samples
354  num_of_samples = buffer->getNumSamples();
355  }
356 
357  // INTERLEAVE all samples together (channel 1 + channel 2 + channel 1 + channel 2, etc...)
358  output = new float[num_of_channels * num_of_samples];
359  int position = 0;
360 
361  // Loop through samples in each channel (combining them)
362  for (int channel = 0; channel < num_of_channels; channel++)
363  {
364  for (int sample = 0; sample < num_of_samples; sample++)
365  {
366  // Add sample to output array
367  output[position] = buffer->getReadPointer(channel)[sample];
368 
369  // increment position
370  position++;
371  }
372  }
373 
374  // Update sample count (since it might have changed due to resampling)
375  *sample_count = num_of_samples;
376 
377  // return combined array
378  return output;
379 }
380 
381 
382 // Get an array of sample data (all channels interleaved together), using any sample rate
383 float* Frame::GetInterleavedAudioSamples(int new_sample_rate, AudioResampler* resampler, int* sample_count)
384 {
385  float *output = NULL;
386  AudioSampleBuffer *buffer(audio.get());
387  int num_of_channels = audio->getNumChannels();
388  int num_of_samples = GetAudioSamplesCount();
389 
390  // Resample to new sample rate (if needed)
391  if (new_sample_rate != sample_rate && resampler)
392  {
393  // YES, RESAMPLE AUDIO
394  resampler->SetBuffer(audio.get(), sample_rate, new_sample_rate);
395 
396  // Resample data, and return new buffer pointer
397  buffer = resampler->GetResampledBuffer();
398 
399  // Update num_of_samples
400  num_of_samples = buffer->getNumSamples();
401  }
402 
403  // INTERLEAVE all samples together (channel 1 + channel 2 + channel 1 + channel 2, etc...)
404  output = new float[num_of_channels * num_of_samples];
405  int position = 0;
406 
407  // Loop through samples in each channel (combining them)
408  for (int sample = 0; sample < num_of_samples; sample++)
409  {
410  for (int channel = 0; channel < num_of_channels; channel++)
411  {
412  // Add sample to output array
413  output[position] = buffer->getReadPointer(channel)[sample];
414 
415  // increment position
416  position++;
417  }
418  }
419 
420  // Update sample count (since it might have changed due to resampling)
421  *sample_count = num_of_samples;
422 
423  // return combined array
424  return output;
425 }
426 
427 // Get number of audio channels
429 {
430  const GenericScopedLock<CriticalSection> lock(addingAudioSection);
431  if (audio)
432  return audio->getNumChannels();
433  else
434  return 0;
435 }
436 
437 // Get number of audio samples
439 {
440  const GenericScopedLock<CriticalSection> lock(addingAudioSection);
441  return max_audio_sample;
442 }
443 
444 juce::AudioSampleBuffer *Frame::GetAudioSampleBuffer()
445 {
446  return audio.get();
447 }
448 
449 // Get the size in bytes of this frame (rough estimate)
451 {
452  int64_t total_bytes = 0;
453  if (image)
454  total_bytes += (width * height * sizeof(char) * 4);
455  if (audio) {
456  // approximate audio size (sample rate / 24 fps)
457  total_bytes += (sample_rate / 24.0) * sizeof(float);
458  }
459 
460  // return size of this frame
461  return total_bytes;
462 }
463 
464 // Get pixel data (as packets)
465 const unsigned char* Frame::GetPixels()
466 {
467  // Check for blank image
468  if (!image)
469  // Fill with black
470  AddColor(width, height, color);
471 
472  // Return array of pixel packets
473  return image->bits();
474 }
475 
476 // Get pixel data (for only a single scan-line)
477 const unsigned char* Frame::GetPixels(int row)
478 {
479  // Return array of pixel packets
480  return image->scanLine(row);
481 }
482 
483 // Set Pixel Aspect Ratio
484 void Frame::SetPixelRatio(int num, int den)
485 {
486  pixel_ratio.num = num;
487  pixel_ratio.den = den;
488 }
489 
490 // Set frame number
491 void Frame::SetFrameNumber(int64_t new_number)
492 {
493  number = new_number;
494 }
495 
496 // Calculate the # of samples per video frame (for a specific frame number and frame rate)
497 int Frame::GetSamplesPerFrame(int64_t number, Fraction fps, int sample_rate, int channels)
498 {
499  // Get the total # of samples for the previous frame, and the current frame (rounded)
500  double fps_rate = fps.Reciprocal().ToDouble();
501 
502  // Determine previous samples total, and make sure it's evenly divisible by the # of channels
503  double previous_samples = (sample_rate * fps_rate) * (number - 1);
504  double previous_samples_remainder = fmod(previous_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
505  previous_samples -= previous_samples_remainder;
506 
507  // Determine the current samples total, and make sure it's evenly divisible by the # of channels
508  double total_samples = (sample_rate * fps_rate) * number;
509  double total_samples_remainder = fmod(total_samples, (double)channels); // subtract the remainder to the total (to make it evenly divisible)
510  total_samples -= total_samples_remainder;
511 
512  // Subtract the previous frame's total samples with this frame's total samples. Not all sample rates can
513  // be evenly divided into frames, so each frame can have have different # of samples.
514  int samples_per_frame = round(total_samples - previous_samples);
515  return samples_per_frame;
516 }
517 
518 // Calculate the # of samples per video frame (for the current frame number)
519 int Frame::GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
520 {
521  return GetSamplesPerFrame(number, fps, sample_rate, channels);
522 }
523 
524 // Get height of image
526 {
527  return height;
528 }
529 
530 // Get height of image
532 {
533  return width;
534 }
535 
536 // Get the original sample rate of this frame's audio data
538 {
539  return sample_rate;
540 }
541 
542 // Get the original sample rate of this frame's audio data
544 {
545  return channel_layout;
546 }
547 
548 
549 // Save the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
550 void Frame::Save(string path, float scale, string format, int quality)
551 {
552  // Get preview image
553  std::shared_ptr<QImage> previewImage = GetImage();
554 
555  // scale image if needed
556  if (abs(scale) > 1.001 || abs(scale) < 0.999)
557  {
558  int new_width = width;
559  int new_height = height;
560 
561  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
562  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
563  {
564  // Calculate correct DAR (display aspect ratio)
565  int new_width = previewImage->size().width();
566  int new_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
567 
568  // Resize to fix DAR
569  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
570  }
571 
572  // Resize image
573  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(new_width * scale, new_height * scale, Qt::KeepAspectRatio, Qt::SmoothTransformation)));
574  }
575 
576  // Save image
577  previewImage->save(QString::fromStdString(path), format.c_str(), quality);
578 }
579 
580 // Thumbnail the frame image to the specified path. The image format is determined from the extension (i.e. image.PNG, image.JPEG)
581 void Frame::Thumbnail(string path, int new_width, int new_height, string mask_path, string overlay_path,
582  string background_color, bool ignore_aspect, string format, int quality, float rotate) {
583 
584  // Create blank thumbnail image & fill background color
585  std::shared_ptr<QImage> thumbnail = std::shared_ptr<QImage>(new QImage(new_width, new_height, QImage::Format_RGBA8888));
586  thumbnail->fill(QColor(QString::fromStdString(background_color)));
587 
588  // Create painter
589  QPainter painter(thumbnail.get());
590  painter.setRenderHints(QPainter::Antialiasing | QPainter::SmoothPixmapTransform | QPainter::TextAntialiasing, true);
591 
592  // Get preview image
593  std::shared_ptr<QImage> previewImage = GetImage();
594 
595  // Update the image to reflect the correct pixel aspect ration (i.e. to fix non-squar pixels)
596  if (pixel_ratio.num != 1 || pixel_ratio.den != 1)
597  {
598  // Calculate correct DAR (display aspect ratio)
599  int aspect_width = previewImage->size().width();
600  int aspect_height = previewImage->size().height() * pixel_ratio.Reciprocal().ToDouble();
601 
602  // Resize to fix DAR
603  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(aspect_width, aspect_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
604  }
605 
606  // Resize frame image
607  if (ignore_aspect)
608  // Ignore aspect ratio
609  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
610  else
611  // Maintain aspect ratio
612  previewImage = std::shared_ptr<QImage>(new QImage(previewImage->scaled(new_width, new_height, Qt::KeepAspectRatio, Qt::SmoothTransformation)));
613 
614  // Composite frame image onto background (centered)
615  int x = (new_width - previewImage->size().width()) / 2.0; // center
616  int y = (new_height - previewImage->size().height()) / 2.0; // center
617  painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
618 
619 
620  // Create transform and rotate (if needed)
621  QTransform transform;
622  float origin_x = previewImage->width() / 2.0;
623  float origin_y = previewImage->height() / 2.0;
624  transform.translate(origin_x, origin_y);
625  transform.rotate(rotate);
626  transform.translate(-origin_x,-origin_y);
627  painter.setTransform(transform);
628 
629  // Draw image onto QImage
630  painter.drawImage(x, y, *previewImage);
631 
632 
633  // Overlay Image (if any)
634  if (overlay_path != "") {
635  // Open overlay
636  std::shared_ptr<QImage> overlay = std::shared_ptr<QImage>(new QImage());
637  overlay->load(QString::fromStdString(overlay_path));
638 
639  // Set pixel format
640  overlay = std::shared_ptr<QImage>(new QImage(overlay->convertToFormat(QImage::Format_RGBA8888)));
641 
642  // Resize to fit
643  overlay = std::shared_ptr<QImage>(new QImage(overlay->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
644 
645  // Composite onto thumbnail
646  painter.setCompositionMode(QPainter::CompositionMode_SourceOver);
647  painter.drawImage(0, 0, *overlay);
648  }
649 
650 
651  // Mask Image (if any)
652  if (mask_path != "") {
653  // Open mask
654  std::shared_ptr<QImage> mask = std::shared_ptr<QImage>(new QImage());
655  mask->load(QString::fromStdString(mask_path));
656 
657  // Set pixel format
658  mask = std::shared_ptr<QImage>(new QImage(mask->convertToFormat(QImage::Format_RGBA8888)));
659 
660  // Resize to fit
661  mask = std::shared_ptr<QImage>(new QImage(mask->scaled(new_width, new_height, Qt::IgnoreAspectRatio, Qt::SmoothTransformation)));
662 
663  // Negate mask
664  mask->invertPixels();
665 
666  // Get pixels
667  unsigned char *pixels = (unsigned char *) thumbnail->bits();
668  unsigned char *mask_pixels = (unsigned char *) mask->bits();
669 
670  // Convert the mask image to grayscale
671  // Loop through pixels
672  for (int pixel = 0, byte_index=0; pixel < new_width * new_height; pixel++, byte_index+=4)
673  {
674  // Get the RGB values from the pixel
675  int gray_value = qGray(mask_pixels[byte_index], mask_pixels[byte_index] + 1, mask_pixels[byte_index] + 2);
676  int Frame_Alpha = pixels[byte_index + 3];
677  int Mask_Value = constrain(Frame_Alpha - gray_value);
678 
679  // Set all alpha pixels to gray value
680  pixels[byte_index + 3] = Mask_Value;
681  }
682  }
683 
684 
685  // End painter
686  painter.end();
687 
688  // Save image
689  thumbnail->save(QString::fromStdString(path), format.c_str(), quality);
690 }
691 
692 // Constrain a color value from 0 to 255
693 int Frame::constrain(int color_value)
694 {
695  // Constrain new color from 0 to 255
696  if (color_value < 0)
697  color_value = 0;
698  else if (color_value > 255)
699  color_value = 255;
700 
701  return color_value;
702 }
703 
704 // Add (or replace) pixel data to the frame (based on a solid color)
705 void Frame::AddColor(int new_width, int new_height, string new_color)
706 {
707  // Set color
708  color = new_color;
709 
710  // Create new image object, and fill with pixel data
711  const GenericScopedLock<CriticalSection> lock(addingImageSection);
712  #pragma omp critical (AddImage)
713  {
714  image = std::shared_ptr<QImage>(new QImage(new_width, new_height, QImage::Format_RGBA8888));
715 
716  // Fill with solid color
717  image->fill(QColor(QString::fromStdString(color)));
718  }
719  // Update height and width
720  width = image->width();
721  height = image->height();
722  has_image_data = true;
723 }
724 
725 // Add (or replace) pixel data to the frame
726 void Frame::AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
727 {
728  // Create new buffer
729  const GenericScopedLock<CriticalSection> lock(addingImageSection);
730  int buffer_size = new_width * new_height * bytes_per_pixel;
731  qbuffer = new unsigned char[buffer_size]();
732 
733  // Copy buffer data
734  memcpy((unsigned char*)qbuffer, pixels_, buffer_size);
735 
736  // Create new image object, and fill with pixel data
737  #pragma omp critical (AddImage)
738  {
739  image = std::shared_ptr<QImage>(new QImage(qbuffer, new_width, new_height, new_width * bytes_per_pixel, type, (QImageCleanupFunction) &openshot::Frame::cleanUpBuffer, (void*) qbuffer));
740 
741  // Always convert to RGBA8888 (if different)
742  if (image->format() != QImage::Format_RGBA8888)
743  *image = image->convertToFormat(QImage::Format_RGBA8888);
744 
745  // Update height and width
746  width = image->width();
747  height = image->height();
748  has_image_data = true;
749  }
750 }
751 
752 // Add (or replace) pixel data to the frame
753 void Frame::AddImage(std::shared_ptr<QImage> new_image)
754 {
755  // Ignore blank images
756  if (!new_image)
757  return;
758 
759  // assign image data
760  const GenericScopedLock<CriticalSection> lock(addingImageSection);
761  #pragma omp critical (AddImage)
762  {
763  image = new_image;
764 
765  // Always convert to RGBA8888 (if different)
766  if (image->format() != QImage::Format_RGBA8888)
767  *image = image->convertToFormat(QImage::Format_RGBA8888);
768 
769  // Update height and width
770  width = image->width();
771  height = image->height();
772  has_image_data = true;
773  }
774 }
775 
776 // Add (or replace) pixel data to the frame (for only the odd or even lines)
777 void Frame::AddImage(std::shared_ptr<QImage> new_image, bool only_odd_lines)
778 {
779  // Ignore blank new_image
780  if (!new_image)
781  return;
782 
783  // Check for blank source image
784  if (!image) {
785  // Replace the blank source image
786  AddImage(new_image);
787 
788  } else {
789 
790  // Ignore image of different sizes or formats
791  bool ret=false;
792  #pragma omp critical (AddImage)
793  if (image == new_image || image->size() != image->size() || image->format() != image->format())
794  ret=true;
795  if (ret)
796  return;
797 
798  // Get the frame's image
799  const GenericScopedLock<CriticalSection> lock(addingImageSection);
800  #pragma omp critical (AddImage)
801  {
802  const unsigned char *pixels = image->bits();
803  const unsigned char *new_pixels = new_image->bits();
804 
805  // Loop through the scanlines of the image (even or odd)
806  int start = 0;
807  if (only_odd_lines)
808  start = 1;
809  for (int row = start; row < image->height(); row += 2) {
810  memcpy((unsigned char *) pixels, new_pixels + (row * image->bytesPerLine()), image->bytesPerLine());
811  new_pixels += image->bytesPerLine();
812  }
813 
814  // Update height and width
815  width = image->width();
816  height = image->height();
817  has_image_data = true;
818  }
819  }
820 }
821 
822 
823 // Resize audio container to hold more (or less) samples and channels
824 void Frame::ResizeAudio(int channels, int length, int rate, ChannelLayout layout)
825 {
826  const GenericScopedLock<CriticalSection> lock(addingAudioSection);
827 
828  // Resize JUCE audio buffer
829  audio->setSize(channels, length, true, true, false);
830  channel_layout = layout;
831  sample_rate = rate;
832 
833  // Calculate max audio sample added
834  max_audio_sample = length;
835 }
836 
837 // Add audio samples to a specific channel
838 void Frame::AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float* source, int numSamples, float gainToApplyToSource = 1.0f) {
839  const GenericScopedLock<CriticalSection> lock(addingAudioSection);
840  #pragma omp critical (adding_audio)
841  {
842  // Clamp starting sample to 0
843  int destStartSampleAdjusted = max(destStartSample, 0);
844 
845  // Extend audio container to hold more (or less) samples and channels.. if needed
846  int new_length = destStartSampleAdjusted + numSamples;
847  int new_channel_length = audio->getNumChannels();
848  if (destChannel >= new_channel_length)
849  new_channel_length = destChannel + 1;
850  if (new_length > audio->getNumSamples() || new_channel_length > audio->getNumChannels())
851  audio->setSize(new_channel_length, new_length, true, true, false);
852 
853  // Clear the range of samples first (if needed)
854  if (replaceSamples)
855  audio->clear(destChannel, destStartSampleAdjusted, numSamples);
856 
857  // Add samples to frame's audio buffer
858  audio->addFrom(destChannel, destStartSampleAdjusted, source, numSamples, gainToApplyToSource);
859  has_audio_data = true;
860 
861  // Calculate max audio sample added
862  if (new_length > max_audio_sample)
863  max_audio_sample = new_length;
864  }
865 }
866 
867 // Apply gain ramp (i.e. fading volume)
868 void Frame::ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain = 0.0f, float final_gain = 1.0f)
869 {
870  const GenericScopedLock<CriticalSection> lock(addingAudioSection);
871 
872  // Apply gain ramp
873  audio->applyGainRamp(destChannel, destStartSample, numSamples, initial_gain, final_gain);
874 }
875 
876 // Get pointer to Magick++ image object
877 std::shared_ptr<QImage> Frame::GetImage()
878 {
879  // Check for blank image
880  if (!image)
881  // Fill with black
882  AddColor(width, height, color);
883 
884  return image;
885 }
886 
887 #ifdef USE_IMAGEMAGICK
888 // Get pointer to ImageMagick image object
889 std::shared_ptr<Magick::Image> Frame::GetMagickImage()
890 {
891  // Check for blank image
892  if (!image)
893  // Fill with black
894  AddColor(width, height, "#000000");
895 
896  // Get the pixels from the frame image
897  QRgb const *tmpBits = (const QRgb*)image->bits();
898 
899  // Create new image object, and fill with pixel data
900  std::shared_ptr<Magick::Image> magick_image = std::shared_ptr<Magick::Image>(new Magick::Image(image->width(), image->height(),"RGBA", Magick::CharPixel, tmpBits));
901 
902  // Give image a transparent background color
903  magick_image->backgroundColor(Magick::Color("none"));
904  magick_image->virtualPixelMethod(Magick::TransparentVirtualPixelMethod);
905  magick_image->matte(true);
906 
907  return magick_image;
908 }
909 #endif
910 
911 #ifdef USE_IMAGEMAGICK
912 // Get pointer to QImage of frame
913 void Frame::AddMagickImage(std::shared_ptr<Magick::Image> new_image)
914 {
915  const int BPP = 4;
916  const std::size_t bufferSize = new_image->columns() * new_image->rows() * BPP;
917 
918  /// Use realloc for fast memory allocation.
919  /// TODO: consider locking the buffer for mt safety
920  //qbuffer = reinterpret_cast<unsigned char*>(realloc(qbuffer, bufferSize));
921  qbuffer = new unsigned char[bufferSize]();
922  unsigned char *buffer = (unsigned char*)qbuffer;
923 
924  // Iterate through the pixel packets, and load our own buffer
925  // Each color needs to be scaled to 8 bit (using the ImageMagick built-in ScaleQuantumToChar function)
926  int numcopied = 0;
927  Magick::PixelPacket *pixels = new_image->getPixels(0,0, new_image->columns(), new_image->rows());
928  for (int n = 0, i = 0; n < new_image->columns() * new_image->rows(); n += 1, i += 4) {
929  buffer[i+0] = MagickCore::ScaleQuantumToChar((Magick::Quantum) pixels[n].red);
930  buffer[i+1] = MagickCore::ScaleQuantumToChar((Magick::Quantum) pixels[n].green);
931  buffer[i+2] = MagickCore::ScaleQuantumToChar((Magick::Quantum) pixels[n].blue);
932  buffer[i+3] = 255 - MagickCore::ScaleQuantumToChar((Magick::Quantum) pixels[n].opacity);
933  numcopied+=4;
934  }
935 
936  // Create QImage of frame data
937  image = std::shared_ptr<QImage>(new QImage(qbuffer, width, height, width * BPP, QImage::Format_RGBA8888, (QImageCleanupFunction) &cleanUpBuffer, (void*) qbuffer));
938 
939  // Update height and width
940  width = image->width();
941  height = image->height();
942  has_image_data = true;
943 }
944 #endif
945 
946 // Play audio samples for this frame
948 {
949  // Check if samples are present
950  if (!GetAudioSamplesCount())
951  return;
952 
953  AudioDeviceManager deviceManager;
954  deviceManager.initialise (0, /* number of input channels */
955  2, /* number of output channels */
956  0, /* no XML settings.. */
957  true /* select default device on failure */);
958  //deviceManager.playTestSound();
959 
960  AudioSourcePlayer audioSourcePlayer;
961  deviceManager.addAudioCallback (&audioSourcePlayer);
962 
963  ScopedPointer<AudioBufferSource> my_source;
964  my_source = new AudioBufferSource(audio.get());
965 
966  // Create TimeSliceThread for audio buffering
967  TimeSliceThread my_thread("Audio buffer thread");
968 
969  // Start thread
970  my_thread.startThread();
971 
972  AudioTransportSource transport1;
973  transport1.setSource (my_source,
974  5000, // tells it to buffer this many samples ahead
975  &my_thread,
976  (double) sample_rate,
977  audio->getNumChannels()); // sample rate of source
978  transport1.setPosition (0);
979  transport1.setGain(1.0);
980 
981 
982  // Create MIXER
983  MixerAudioSource mixer;
984  mixer.addInputSource(&transport1, false);
985  audioSourcePlayer.setSource (&mixer);
986 
987  // Start transports
988  transport1.start();
989 
990  while (transport1.isPlaying())
991  {
992  cout << "playing" << endl;
993  usleep(1000000);
994  }
995 
996  cout << "DONE!!!" << endl;
997 
998  transport1.stop();
999  transport1.setSource (0);
1000  audioSourcePlayer.setSource (0);
1001  my_thread.stopThread(500);
1002  deviceManager.removeAudioCallback (&audioSourcePlayer);
1003  deviceManager.closeAudioDevice();
1004  deviceManager.removeAllChangeListeners();
1005  deviceManager.dispatchPendingMessages();
1006 
1007  cout << "End of Play()" << endl;
1008 
1009 
1010 }
1011 
1012 // Clean up buffer after QImage is deleted
1013 void Frame::cleanUpBuffer(void *info)
1014 {
1015  if (info)
1016  {
1017  // Remove buffer since QImage tells us to
1018  unsigned char* ptr_to_qbuffer = (unsigned char*) info;
1019  delete[] ptr_to_qbuffer;
1020  }
1021 }
1022 
1023 // Add audio silence
1024 void Frame::AddAudioSilence(int numSamples)
1025 {
1026  const GenericScopedLock<CriticalSection> lock(addingAudioSection);
1027 
1028  // Resize audio container
1029  audio->setSize(channels, numSamples, false, true, false);
1030  audio->clear();
1031  has_audio_data = true;
1032 
1033  // Calculate max audio sample added
1034  if (numSamples > max_audio_sample)
1035  max_audio_sample = numSamples;
1036 }
void SetBuffer(AudioSampleBuffer *new_buffer, double sample_rate, double new_sample_rate)
Sets the audio buffer and key settings.
int GetWidth()
Get height of image.
Definition: Frame.cpp:531
int num
Numerator for the fraction.
Definition: Fraction.h:44
int GetAudioSamplesCount()
Get number of audio samples.
Definition: Frame.cpp:438
float * GetInterleavedAudioSamples(int new_sample_rate, AudioResampler *resampler, int *sample_count)
Get an array of sample data (all channels interleaved together), using any sample rate...
Definition: Frame.cpp:383
This class represents a single frame of video (i.e. image & audio data)
Definition: Frame.h:115
const unsigned char * GetWaveformPixels(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image pixels.
Definition: Frame.cpp:274
juce::AudioSampleBuffer * GetAudioSampleBuffer()
Definition: Frame.cpp:444
const unsigned char * GetPixels()
Get pixel data (as packets)
Definition: Frame.cpp:465
void Play()
Play audio samples for this frame.
Definition: Frame.cpp:947
void Save(string path, float scale, string format="PNG", int quality=100)
Save the frame image to the specified path. The image format can be BMP, JPG, JPEG, PNG, PPM, XBM, XPM.
Definition: Frame.cpp:550
void DeepCopy(const Frame &other)
Copy data and pointers from another Frame instance.
Definition: Frame.cpp:102
void AddAudio(bool replaceSamples, int destChannel, int destStartSample, const float *source, int numSamples, float gainToApplyToSource)
Add audio samples to a specific channel.
Definition: Frame.cpp:838
void Display()
Display the frame image to the screen (primarily used for debugging reasons)
Definition: Frame.cpp:131
Fraction Reciprocal()
Return the reciprocal as a Fraction.
Definition: Fraction.cpp:81
int64_t number
This is the frame number (starting at 1)
Definition: Frame.h:138
This class is used to expose an AudioSampleBuffer as an AudioSource in JUCE.
Frame & operator=(const Frame &other)
Assignment operator.
Definition: Frame.cpp:93
void ClearWaveform()
Clear the waveform image (and deallocate it's memory)
Definition: Frame.cpp:267
void AddColor(int new_width, int new_height, string new_color)
Add (or replace) pixel data to the frame (based on a solid color)
Definition: Frame.cpp:705
float * GetAudioSamples(int channel)
Get an array of sample data.
Definition: Frame.cpp:330
void SetFrameNumber(int64_t number)
Set frame number.
Definition: Frame.cpp:491
void AddAudioSilence(int numSamples)
Add audio silence.
Definition: Frame.cpp:1024
bool has_audio_data
This frame has been loaded with audio data.
Definition: Frame.h:139
This class represents a fraction.
Definition: Fraction.h:42
void Thumbnail(string path, int new_width, int new_height, string mask_path, string overlay_path, string background_color, bool ignore_aspect, string format="png", int quality=100, float rotate=0.0)
Definition: Frame.cpp:581
static void cleanUpBuffer(void *info)
Clean up buffer after QImage is deleted.
Definition: Frame.cpp:1013
ChannelLayout
This enumeration determines the audio channel layout (such as stereo, mono, 5 point surround...
void AddImage(int new_width, int new_height, int bytes_per_pixel, QImage::Format type, const unsigned char *pixels_)
Add (or replace) pixel data to the frame.
Definition: Frame.cpp:726
Frame()
Constructor - blank frame (300x200 blank image, 48kHz audio silence)
Definition: Frame.cpp:34
void ApplyGainRamp(int destChannel, int destStartSample, int numSamples, float initial_gain, float final_gain)
Apply gain ramp (i.e. fading volume)
Definition: Frame.cpp:868
void DisplayWaveform()
Display the wave form.
Definition: Frame.cpp:284
void SetPixelRatio(int num, int den)
Set Pixel Aspect Ratio.
Definition: Frame.cpp:484
int GetAudioChannelsCount()
Get number of audio channels.
Definition: Frame.cpp:428
This namespace is the default namespace for all code in the openshot library.
ChannelLayout ChannelsLayout()
Definition: Frame.cpp:543
std::shared_ptr< QImage > GetImage()
Get pointer to Qt QImage image object.
Definition: Frame.cpp:877
std::shared_ptr< QImage > GetWaveform(int width, int height, int Red, int Green, int Blue, int Alpha)
Get an audio waveform image.
Definition: Frame.cpp:172
int64_t GetBytes()
Get the size in bytes of this frame (rough estimate)
Definition: Frame.cpp:450
~Frame()
Destructor.
Definition: Frame.cpp:124
AudioSampleBuffer * GetResampledBuffer()
Get the resampled audio buffer.
float * GetPlanarAudioSamples(int new_sample_rate, AudioResampler *resampler, int *sample_count)
Definition: Frame.cpp:337
int den
Denominator for the fraction.
Definition: Fraction.h:45
float GetAudioSample(int channel, int sample, int magnitude_range)
Get magnitude of range of samples (if channel is -1, return average of all channels for that sample) ...
Definition: Frame.cpp:317
int GetSamplesPerFrame(Fraction fps, int sample_rate, int channels)
Calculate the # of samples per video frame (for the current frame number)
Definition: Frame.cpp:519
bool has_image_data
This frame has been loaded with pixel data.
Definition: Frame.h:140
void ResizeAudio(int channels, int length, int sample_rate, ChannelLayout channel_layout)
Resize audio container to hold more (or less) samples and channels.
Definition: Frame.cpp:824
int GetHeight()
Get height of image.
Definition: Frame.cpp:525
double ToDouble()
Return this fraction as a double (i.e. 1/2 = 0.5)
Definition: Fraction.cpp:46
int SampleRate()
Get the original sample rate of this frame's audio data.
Definition: Frame.cpp:537
This class is used to resample audio data for many sequential frames.