Merge pull request #4785 from Laserlicht/subtitles

Subtitles for videos / sync

client/media/CSoundHandler.cpp

@@ -240,6 +240,18 @@ void CSoundHandler::stopSound(int handler)
 		Mix_HaltChannel(handler);
 }
 
+void CSoundHandler::pauseSound(int handler)
+{
+	if(isInitialized() && handler != -1)
+		Mix_Pause(handler);
+}
+
+void CSoundHandler::resumeSound(int handler)
+{
+	if(isInitialized() && handler != -1)
+		Mix_Resume(handler);
+}
+
 ui32 CSoundHandler::getVolume() const
 {
 	return volume;

client/media/CSoundHandler.h

@@ -67,6 +67,8 @@ public:
 	int playSound(std::pair<std::unique_ptr<ui8[]>, si64> & data, int repeats = 0, bool cache = false) final;
 	int playSoundFromSet(std::vector<soundBase::soundID> & sound_vec) final;
 	void stopSound(int handler) final;
+	void pauseSound(int handler) final;
+	void resumeSound(int handler) final;
 
 	void setCallback(int channel, std::function<void()> function) final;
 	void resetCallback(int channel) final;

client/media/CVideoHandler.cpp

@@ -316,6 +316,12 @@ bool CVideoInstance::loadNextFrame()
 	return true;
 }
 
+
+double CVideoInstance::timeStamp()
+{
+	return getCurrentFrameEndTime();
+}
+
 bool CVideoInstance::videoEnded()
 {
 	return getCurrentFrame() == nullptr;
@@ -385,12 +391,38 @@ void CVideoInstance::tick(uint32_t msPassed)
 	if(videoEnded())
 		throw std::runtime_error("Video already ended!");
 
-	frameTime += msPassed / 1000.0;
+	if(startTime == std::chrono::steady_clock::time_point())
+		startTime = std::chrono::steady_clock::now();
 
-	if(frameTime >= getCurrentFrameEndTime())
+	auto nowTime = std::chrono::steady_clock::now();
+	double difference = std::chrono::duration_cast<std::chrono::milliseconds>(nowTime - startTime).count() / 1000.0;
+
+	int frameskipCounter = 0;
+	while(!videoEnded() && difference >= getCurrentFrameEndTime() + getCurrentFrameDuration() && frameskipCounter < MAX_FRAMESKIP) // Frameskip
+	{
+		decodeNextFrame();
+		frameskipCounter++;
+	}
+	if(!videoEnded() && difference >= getCurrentFrameEndTime())
 		loadNextFrame();
 }
 
+
+void CVideoInstance::activate()
+{
+	if(deactivationStartTime != std::chrono::steady_clock::time_point())
+	{
+		auto pauseDuration = std::chrono::steady_clock::now() - deactivationStartTime;
+		startTime += pauseDuration;
+		deactivationStartTime = std::chrono::steady_clock::time_point();
+	}
+}
+
+void CVideoInstance::deactivate()
+{
+	deactivationStartTime = std::chrono::steady_clock::now();
+}
+
 struct FFMpegFormatDescription
 {
 	uint8_t sampleSizeBytes;

client/media/CVideoHandler.h

@@ -77,22 +77,28 @@ class CVideoInstance final : public IVideoInstance, public FFMpegStream
 	SDL_Surface * surface = nullptr;
 	Point dimensions;
 
-	/// video playback current progress, in seconds
-	double frameTime = 0.0;
+	/// video playback start time point
+	std::chrono::steady_clock::time_point startTime;
+	std::chrono::steady_clock::time_point deactivationStartTime;
 
 	void prepareOutput(float scaleFactor, bool useTextureOutput);
 	
+	const int MAX_FRAMESKIP = 5;
+
 public:
 	~CVideoInstance();
 
 	void openVideo();
 	bool loadNextFrame();
 
+	double timeStamp() final;
 	bool videoEnded() final;
 	Point size() final;
 
 	void show(const Point & position, Canvas & canvas) final;
 	void tick(uint32_t msPassed) final;
+	void activate() final;
+	void deactivate() final;
 };
 
 class CVideoPlayer final : public IVideoPlayer

client/media/ISoundPlayer.h

@@ -22,6 +22,8 @@ public:
 	virtual int playSound(std::pair<std::unique_ptr<ui8[]>, si64> & data, int repeats = 0, bool cache = false) = 0;
 	virtual int playSoundFromSet(std::vector<soundBase::soundID> & sound_vec) = 0;
 	virtual void stopSound(int handler) = 0;
+	virtual void pauseSound(int handler) = 0;
+	virtual void resumeSound(int handler) = 0;
 
 	virtual ui32 getVolume() const = 0;
 	virtual void setVolume(ui32 percent) = 0;

client/media/IVideoPlayer.h

@@ -20,6 +20,9 @@ VCMI_LIB_NAMESPACE_END
 class IVideoInstance
 {
 public:
+	/// Returns current video timestamp
+	virtual double timeStamp() = 0;
+
 	/// Returns true if video playback is over
 	virtual bool videoEnded() = 0;
 
@@ -32,6 +35,10 @@ public:
 	/// Advances video playback by specified duration
 	virtual void tick(uint32_t msPassed) = 0;
 
+	/// activate or deactivate video
+	virtual void activate() = 0;
+	virtual void deactivate() = 0;
+
 	virtual ~IVideoInstance() = default;
 };
 

client/widgets/VideoWidget.cpp

@@ -9,6 +9,7 @@
  */
 #include "StdInc.h"
 #include "VideoWidget.h"
+#include "TextControls.h"
 
 #include "../CGameInfo.h"
 #include "../gui/CGuiHandler.h"
@@ -16,6 +17,8 @@
 #include "../media/IVideoPlayer.h"
 #include "../render/Canvas.h"
 
+#include "../../lib/filesystem/Filesystem.h"
+
 VideoWidgetBase::VideoWidgetBase(const Point & position, const VideoPath & video, bool playAudio)
 	: VideoWidgetBase(position, video, playAudio, 1.0)
 {
@@ -33,11 +36,22 @@ VideoWidgetBase::~VideoWidgetBase() = default;
 
 void VideoWidgetBase::playVideo(const VideoPath & fileToPlay)
 {
+	OBJECT_CONSTRUCTION;
+
+	JsonPath subTitlePath = fileToPlay.toType<EResType::JSON>();
+	JsonPath subTitlePathVideoDir = subTitlePath.addPrefix("VIDEO/");
+	if(CResourceHandler::get()->existsResource(subTitlePath))
+		subTitleData = JsonNode(subTitlePath);
+	else if(CResourceHandler::get()->existsResource(subTitlePathVideoDir))
+		subTitleData = JsonNode(subTitlePathVideoDir);
+
 	videoInstance = CCS->videoh->open(fileToPlay, scaleFactor);
 	if (videoInstance)
 	{
 		pos.w = videoInstance->size().x;
 		pos.h = videoInstance->size().y;
+		if(!subTitleData.isNull())
+			subTitle = std::make_unique<CMultiLineLabel>(Rect(0, (pos.h / 5) * 4, pos.w, pos.h / 5), EFonts::FONT_HIGH_SCORE, ETextAlignment::CENTER, Colors::WHITE);
 	}
 
 	if (playAudio)
@@ -52,6 +66,8 @@ void VideoWidgetBase::show(Canvas & to)
 {
 	if(videoInstance)
 		videoInstance->show(pos.topLeft(), to);
+	if(subTitle)
+		subTitle->showAll(to);
 }
 
 void VideoWidgetBase::loadAudio(const VideoPath & fileToPlay)
@@ -91,22 +107,43 @@ void VideoWidgetBase::stopAudio()
 	}
 }
 
+std::string VideoWidgetBase::getSubTitleLine(double timestamp)
+{
+	if(subTitleData.isNull())
+		return {};
+
+	for(auto & segment : subTitleData.Vector())
+		if(timestamp > segment["timeStart"].Float() && timestamp < segment["timeEnd"].Float())
+			return segment["text"].String();
+	
+	return {};
+}
+
 void VideoWidgetBase::activate()
 {
 	CIntObject::activate();
+	if(audioHandle != -1)
+		CCS->soundh->resumeSound(audioHandle);
+	else
 		startAudio();
+	if(videoInstance)
+		videoInstance->activate();
 }
 
 void VideoWidgetBase::deactivate()
 {
 	CIntObject::deactivate();
-	stopAudio();
+	CCS->soundh->pauseSound(audioHandle);
+	if(videoInstance)
+		videoInstance->deactivate();
 }
 
 void VideoWidgetBase::showAll(Canvas & to)
 {
 	if(videoInstance)
 		videoInstance->show(pos.topLeft(), to);
+	if(subTitle)
+		subTitle->showAll(to);
 }
 
 void VideoWidgetBase::tick(uint32_t msPassed)
@@ -122,6 +159,8 @@ void VideoWidgetBase::tick(uint32_t msPassed)
 			onPlaybackFinished();
 		}
 	}
+	if(subTitle && videoInstance)
+		subTitle->setText(getSubTitleLine(videoInstance->timeStamp()));
 }
 
 VideoWidget::VideoWidget(const Point & position, const VideoPath & prologue, const VideoPath & looped, bool playAudio)

client/widgets/VideoWidget.h

@@ -12,21 +12,26 @@
 #include "../gui/CIntObject.h"
 
 #include "../lib/filesystem/ResourcePath.h"
+#include "../lib/json/JsonNode.h"
 
 class IVideoInstance;
+class CMultiLineLabel;
 
 class VideoWidgetBase : public CIntObject
 {
 	std::unique_ptr<IVideoInstance> videoInstance;
+	std::unique_ptr<CMultiLineLabel> subTitle;
 
 	std::pair<std::unique_ptr<ui8[]>, si64> audioData = {nullptr, 0};
 	int audioHandle = -1;
 	bool playAudio = false;
 	float scaleFactor = 1.0;
+	JsonNode subTitleData;
 
 	void loadAudio(const VideoPath & file);
 	void startAudio();
 	void stopAudio();
+	std::string getSubTitleLine(double timestamp);
 
 protected:
 	VideoWidgetBase(const Point & position, const VideoPath & video, bool playAudio);

docs/translators/Translations.md

@@ -56,6 +56,19 @@ This will export all strings from game into `Documents/My Games/VCMI/extracted/t
 
 To export maps and campaigns, use '/translate maps' command instead.
 
+### Video subtitles
+It's possible to add video subtitles. Create a JSON file in `video` folder of translation mod with the name of the video (e.g. `H3Intro.json`):
+```
+[
+    {
+        "timeStart" : 5.640, // start time, seconds
+        "timeEnd" : 8.120, // end time, seconds
+        "text" : " ... " // text to show during this period
+    },
+    ...
+]
+```
+
 ## Translating VCMI data
 
 VCMI contains several new strings, to cover functionality not existing in Heroes III. It can be roughly split into following parts: