Merge pull request #3023 from Laserlicht/video_audio

audio from SMK video file

client/CMT.cpp

@@ -423,11 +423,20 @@ int main(int argc, char * argv[])
 //plays intro, ends when intro is over or button has been pressed (handles events)
 void playIntro()
 {
+	auto audioData = CCS->videoh->getAudio(VideoPath::builtin("3DOLOGO.SMK"));
+	int sound = CCS->soundh->playSound(audioData);
 	if(CCS->videoh->openAndPlayVideo(VideoPath::builtin("3DOLOGO.SMK"), 0, 1, true, true))
 	{
+		audioData = CCS->videoh->getAudio(VideoPath::builtin("NWCLOGO.SMK"));
+		sound = CCS->soundh->playSound(audioData);
 		if (CCS->videoh->openAndPlayVideo(VideoPath::builtin("NWCLOGO.SMK"), 0, 1, true, true))
+		{
+			audioData = CCS->videoh->getAudio(VideoPath::builtin("H3INTRO.SMK"));
+			sound = CCS->soundh->playSound(audioData);
 			CCS->videoh->openAndPlayVideo(VideoPath::builtin("H3INTRO.SMK"), 0, 1, true, true);
+		}
 	}
+	CCS->soundh->stopSound(sound);
 }
 
 static void mainLoop()

client/CMusicHandler.cpp

@@ -142,6 +142,30 @@ Mix_Chunk *CSoundHandler::GetSoundChunk(const AudioPath & sound, bool cache)
 	}
 }
 
+Mix_Chunk *CSoundHandler::GetSoundChunk(std::pair<std::unique_ptr<ui8 []>, si64> & data, bool cache)
+{
+	try
+	{
+		std::vector<ui8> startBytes = std::vector<ui8>(data.first.get(), data.first.get() + std::min((si64)100, data.second));
+
+		if (cache && soundChunksRaw.find(startBytes) != soundChunksRaw.end())
+			return soundChunksRaw[startBytes].first;
+
+		SDL_RWops *ops = SDL_RWFromMem(data.first.get(), (int)data.second);
+		Mix_Chunk *chunk = Mix_LoadWAV_RW(ops, 1);	// will free ops
+
+		if (cache)
+			soundChunksRaw.insert({startBytes, std::make_pair (chunk, std::move (data.first))});
+
+		return chunk;
+	}
+	catch(std::exception &e)
+	{
+		logGlobal->warn("Cannot get sound chunk: %s", e.what());
+		return nullptr;
+	}
+}
+
 int CSoundHandler::ambientDistToVolume(int distance) const
 {
 	const auto & distancesVector = ambientConfig["distances"].Vector();
@@ -197,6 +221,26 @@ int CSoundHandler::playSound(const AudioPath & sound, int repeats, bool cache)
 	return channel;
 }
 
+int CSoundHandler::playSound(std::pair<std::unique_ptr<ui8 []>, si64> & data, int repeats, bool cache)
+{
+	int channel = -1;
+	if (Mix_Chunk *chunk = GetSoundChunk(data, cache))
+	{
+		channel = Mix_PlayChannel(-1, chunk, repeats);
+		if (channel == -1)
+		{
+			logGlobal->error("Unable to play sound, error %s", Mix_GetError());
+			if (!cache)
+				Mix_FreeChunk(chunk);
+		}
+		else if (cache)
+			initCallback(channel);
+		else
+			initCallback(channel, [chunk](){ Mix_FreeChunk(chunk);});
+	}
+	return channel;
+}
+
 // Helper. Randomly select a sound from an array and play it
 int CSoundHandler::playSoundFromSet(std::vector<soundBase::soundID> &sound_vec)
 {

client/CMusicHandler.h

@@ -41,8 +41,10 @@ private:
 
 	using CachedChunk = std::pair<Mix_Chunk *, std::unique_ptr<ui8[]>>;
 	std::map<AudioPath, CachedChunk> soundChunks;
+	std::map<std::vector<ui8>, CachedChunk> soundChunksRaw;
 
 	Mix_Chunk *GetSoundChunk(const AudioPath & sound, bool cache);
+	Mix_Chunk *GetSoundChunk(std::pair<std::unique_ptr<ui8 []>, si64> & data, bool cache);
 
 	/// have entry for every currently active channel
 	/// vector will be empty if callback was not set
@@ -76,6 +78,7 @@ public:
 	// Sounds
 	int playSound(soundBase::soundID soundID, int repeats=0);
 	int playSound(const AudioPath & sound, int repeats=0, bool cache=false);
+	int playSound(std::pair<std::unique_ptr<ui8 []>, si64> & data, int repeats=0, bool cache=false);
 	int playSoundFromSet(std::vector<soundBase::soundID> &sound_vec);
 	void stopSound(int handler);
 

client/CVideoHandler.cpp

@@ -55,6 +55,24 @@ static si64 lodSeek(void * opaque, si64 pos, int whence)
 	return video->data->seek(pos);
 }
 
+// Define a set of functions to read data
+static int lodReadAudio(void* opaque, uint8_t* buf, int size)
+{
+	auto video = reinterpret_cast<CVideoPlayer *>(opaque);
+
+	return static_cast<int>(video->dataAudio->read(buf, size));
+}
+
+static si64 lodSeekAudio(void * opaque, si64 pos, int whence)
+{
+	auto video = reinterpret_cast<CVideoPlayer *>(opaque);
+
+	if (whence & AVSEEK_SIZE)
+		return video->dataAudio->getSize();
+
+	return video->dataAudio->seek(pos);
+}
+
 CVideoPlayer::CVideoPlayer()
 	: stream(-1)
 	, format (nullptr)
@@ -435,6 +453,162 @@ void CVideoPlayer::close()
 	}
 }
 
+std::pair<std::unique_ptr<ui8 []>, si64> CVideoPlayer::getAudio(const VideoPath & videoToOpen)
+{
+	std::pair<std::unique_ptr<ui8 []>, si64> dat(std::make_pair(nullptr, 0));
+
+	VideoPath fnameAudio;
+
+	if (CResourceHandler::get()->existsResource(videoToOpen))
+		fnameAudio = videoToOpen;
+	else
+		fnameAudio = videoToOpen.addPrefix("VIDEO/");
+
+	if (!CResourceHandler::get()->existsResource(fnameAudio))
+	{
+		logGlobal->error("Error: video %s was not found", fnameAudio.getName());
+		return dat;
+	}
+
+	dataAudio = CResourceHandler::get()->load(fnameAudio);
+
+	static const int BUFFER_SIZE = 4096;
+
+	unsigned char * bufferAudio  = (unsigned char *)av_malloc(BUFFER_SIZE);// will be freed by ffmpeg
+	AVIOContext * contextAudio = avio_alloc_context( bufferAudio, BUFFER_SIZE, 0, (void *)this, lodReadAudio, nullptr, lodSeekAudio);
+
+	AVFormatContext * formatAudio = avformat_alloc_context();
+	formatAudio->pb = contextAudio;
+	// filename is not needed - file was already open and stored in this->data;
+	int avfopen = avformat_open_input(&formatAudio, "dummyFilename", nullptr, nullptr);
+
+	if (avfopen != 0)
+	{
+		return dat;
+	}
+	// Retrieve stream information
+	if (avformat_find_stream_info(formatAudio, nullptr) < 0)
+		return dat;
+
+	// Find the first audio stream
+	int streamAudio = -1;
+	for(ui32 i = 0; i < formatAudio->nb_streams; i++)
+	{
+		if (formatAudio->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO)
+		{
+			streamAudio = i;
+			break;
+		}
+	}
+
+	if(streamAudio < 0)
+		return dat;
+
+	const AVCodec *codecAudio = avcodec_find_decoder(formatAudio->streams[streamAudio]->codecpar->codec_id);
+		
+	AVCodecContext *codecContextAudio;
+	if (codecAudio != nullptr)
+		codecContextAudio = avcodec_alloc_context3(codecAudio);
+
+	// Get a pointer to the codec context for the audio stream
+	if (streamAudio > -1)
+	{
+		int ret = avcodec_parameters_to_context(codecContextAudio, formatAudio->streams[streamAudio]->codecpar);
+		if (ret < 0)
+		{
+			//We cannot get codec from parameters
+			avcodec_free_context(&codecContextAudio);
+		}
+	}
+	
+	// Open codec
+	AVFrame *frameAudio;
+	if (codecAudio != nullptr)
+	{
+		if ( avcodec_open2(codecContextAudio, codecAudio, nullptr) < 0 )
+		{
+			// Could not open codec
+			codecAudio = nullptr;
+		}
+		// Allocate audio frame
+		frameAudio = av_frame_alloc();
+	}
+		
+	AVPacket packet;
+
+	std::vector<ui8> samples;
+
+	while (av_read_frame(formatAudio, &packet) >= 0)
+	{
+		if(packet.stream_index == streamAudio)
+		{
+			int rc = avcodec_send_packet(codecContextAudio, &packet);
+			if (rc >= 0)
+				packet.size = 0;
+			rc = avcodec_receive_frame(codecContextAudio, frameAudio);
+			int bytesToRead = (frameAudio->nb_samples * 2 * (formatAudio->streams[streamAudio]->codecpar->bits_per_coded_sample / 8));
+			if (rc >= 0)
+				for (int s = 0; s < bytesToRead; s += sizeof(ui8))
+				{
+					ui8 value;
+					memcpy(&value, &frameAudio->data[0][s], sizeof(ui8));
+					samples.push_back(value);
+				}
+		}
+
+		av_packet_unref(&packet);
+	}
+
+	typedef struct WAV_HEADER {
+		ui8 RIFF[4] = {'R', 'I', 'F', 'F'};
+		ui32 ChunkSize;
+		ui8 WAVE[4] = {'W', 'A', 'V', 'E'};
+		ui8 fmt[4] = {'f', 'm', 't', ' '};
+		ui32 Subchunk1Size = 16;
+		ui16 AudioFormat = 1;
+		ui16 NumOfChan = 2;
+		ui32 SamplesPerSec = 22050;
+		ui32 bytesPerSec = 22050 * 2;
+		ui16 blockAlign = 2;
+		ui16 bitsPerSample = 16;
+		ui8 Subchunk2ID[4] = {'d', 'a', 't', 'a'};
+		ui32 Subchunk2Size;
+	} wav_hdr;
+
+	wav_hdr wav;
+	wav.ChunkSize = samples.size() + sizeof(wav_hdr) - 8;
+  	wav.Subchunk2Size = samples.size() + sizeof(wav_hdr) - 44;
+	wav.SamplesPerSec = formatAudio->streams[streamAudio]->codecpar->sample_rate;
+	wav.bitsPerSample = formatAudio->streams[streamAudio]->codecpar->bits_per_coded_sample;
+	auto wavPtr = reinterpret_cast<ui8*>(&wav);
+
+	dat = std::make_pair(std::make_unique<ui8[]>(samples.size() + sizeof(wav_hdr)), samples.size() + sizeof(wav_hdr));
+	std::copy(wavPtr, wavPtr + sizeof(wav_hdr), dat.first.get());
+	std::copy(samples.begin(), samples.end(), dat.first.get() + sizeof(wav_hdr));
+
+	if (frameAudio)
+		av_frame_free(&frameAudio);
+
+	if (codecAudio)
+	{
+		avcodec_close(codecContextAudio);
+		codecAudio = nullptr;
+	}
+	if (codecContextAudio)
+		avcodec_free_context(&codecContextAudio);
+
+	if (formatAudio)
+		avformat_close_input(&formatAudio);
+
+	if (contextAudio)
+	{
+		av_free(contextAudio);
+		contextAudio = nullptr;
+	}
+
+	return dat;
+}
+
 // Plays a video. Only works for overlays.
 bool CVideoPlayer::playVideo(int x, int y, bool stopOnKey)
 {

client/CVideoHandler.h

@@ -36,6 +36,7 @@ public:
 	{
 		return false;
 	}
+	virtual std::pair<std::unique_ptr<ui8 []>, si64> getAudio(const VideoPath & videoToOpen) { return std::make_pair(nullptr, 0); };
 };
 
 class CEmptyVideoPlayer : public IMainVideoPlayer
@@ -89,7 +90,6 @@ class CVideoPlayer : public IMainVideoPlayer
 
 	bool playVideo(int x, int y, bool stopOnKey);
 	bool open(const VideoPath & fname, bool loop, bool useOverlay = false, bool scale = false);
-
 public:
 	CVideoPlayer();
 	~CVideoPlayer();
@@ -106,6 +106,8 @@ public:
 	// Opens video, calls playVideo, closes video; returns playVideo result (if whole video has been played)
 	bool openAndPlayVideo(const VideoPath & name, int x, int y, bool stopOnKey = false, bool scale = false) override;
 
+	std::pair<std::unique_ptr<ui8 []>, si64> getAudio(const VideoPath & videoToOpen) override;
+
 	//TODO:
 	bool wait() override {return false;};
 	int curFrame() const override {return -1;};
@@ -113,6 +115,7 @@ public:
 
 	// public to allow access from ffmpeg IO functions
 	std::unique_ptr<CInputStream> data;
+	std::unique_ptr<CInputStream> dataAudio;
 };
 
 #endif

client/mainmenu/CHighScoreScreen.cpp

@@ -215,7 +215,7 @@ void CHighScoreScreen::buttonExitClick()
 }
 
 CHighScoreInputScreen::CHighScoreInputScreen(bool won, HighScoreCalculation calc)
-	: CWindowObject(BORDERED), won(won), calc(calc)
+	: CWindowObject(BORDERED), won(won), calc(calc), videoSoundHandle(-1)
 {
 	addUsedEvents(LCLICK | KEYBOARD);
 
@@ -295,6 +295,8 @@ void CHighScoreInputScreen::show(Canvas & to)
 		{
 			CCS->videoh->close();
 			video = "HSLOOP.SMK";
+			auto audioData = CCS->videoh->getAudio(VideoPath::builtin(video));
+			videoSoundHandle = CCS->soundh->playSound(audioData);
 			CCS->videoh->open(VideoPath::builtin(video));
 		}
 		else
@@ -307,6 +309,8 @@ void CHighScoreInputScreen::show(Canvas & to)
 
 void CHighScoreInputScreen::activate()
 {
+	auto audioData = CCS->videoh->getAudio(VideoPath::builtin(video));
+	videoSoundHandle = CCS->soundh->playSound(audioData);
 	if(!CCS->videoh->open(VideoPath::builtin(video)))
 	{
 		if(!won)
@@ -320,6 +324,7 @@ void CHighScoreInputScreen::activate()
 void CHighScoreInputScreen::deactivate()
 {
 	CCS->videoh->close();
+	CCS->soundh->stopSound(videoSoundHandle);
 	CIntObject::deactivate();
 }
 

client/mainmenu/CHighScoreScreen.h

@@ -95,6 +95,7 @@ class CHighScoreInputScreen : public CWindowObject
 	std::shared_ptr<TransparentFilledRectangle> background;
 
 	std::string video;
+	int videoSoundHandle;
 	bool won;
 	HighScoreCalculation calc;
 public:

client/mainmenu/CPrologEpilogVideo.cpp

@@ -20,13 +20,15 @@
 
 
 CPrologEpilogVideo::CPrologEpilogVideo(CampaignScenarioPrologEpilog _spe, std::function<void()> callback)
-	: CWindowObject(BORDERED), spe(_spe), positionCounter(0), voiceSoundHandle(-1), exitCb(callback)
+	: CWindowObject(BORDERED), spe(_spe), positionCounter(0), voiceSoundHandle(-1), videoSoundHandle(-1), exitCb(callback)
 {
 	OBJ_CONSTRUCTION_CAPTURING_ALL_NO_DISPOSE;
 	addUsedEvents(LCLICK);
 	pos = center(Rect(0, 0, 800, 600));
 	updateShadow();
 
+	auto audioData = CCS->videoh->getAudio(spe.prologVideo);
+	videoSoundHandle = CCS->soundh->playSound(audioData);
 	CCS->videoh->open(spe.prologVideo);
 	CCS->musich->playMusic(spe.prologMusic, true, true);
 	voiceSoundHandle = CCS->soundh->playSound(spe.prologVoice);
@@ -63,5 +65,6 @@ void CPrologEpilogVideo::clickPressed(const Point & cursorPosition)
 {
 	close();
 	CCS->soundh->stopSound(voiceSoundHandle);
+	CCS->soundh->stopSound(videoSoundHandle);
 	exitCb();
 }

client/mainmenu/CPrologEpilogVideo.h

@@ -19,6 +19,7 @@ class CPrologEpilogVideo : public CWindowObject
 	CampaignScenarioPrologEpilog spe;
 	int positionCounter;
 	int voiceSoundHandle;
+	int videoSoundHandle;
 	std::function<void()> exitCb;
 
 	std::shared_ptr<CMultiLineLabel> text;