diff --git a/client/lobby/CLobbyScreen.cpp b/client/lobby/CLobbyScreen.cpp
index ad076088a..4406f2865 100644
--- a/client/lobby/CLobbyScreen.cpp
+++ b/client/lobby/CLobbyScreen.cpp
@@ -187,9 +187,19 @@ void CLobbyScreen::updateAfterStateChange()
 		tabOpt->recreate();
 
 	card->changeSelection();
-	if(card->iconDifficulty)
-		card->iconDifficulty->setSelected(CSH->si->difficulty);
-
+	if (card->iconDifficulty)
+	{
+		if (screenType == ESelectionScreen::loadGame)
+		{
+			// When loading the game, only one button in the difficulty toggle group should be enabled, so here disable all other buttons first, then make selection
+			card->iconDifficulty->setSelectedOnly(CSH->si->difficulty);
+		}
+		else
+		{
+			card->iconDifficulty->setSelected(CSH->si->difficulty);
+		}
+	}
+	
 	if(curTab == tabRand && CSH->si->mapGenOptions)
 		tabRand->setMapGenOptions(CSH->si->mapGenOptions);
 }
diff --git a/client/widgets/Buttons.cpp b/client/widgets/Buttons.cpp
index f605866fd..d50e3cf0a 100644
--- a/client/widgets/Buttons.cpp
+++ b/client/widgets/Buttons.cpp
@@ -322,6 +322,11 @@ void CToggleBase::doSelect(bool on)
 	// for overrides
 }
 
+void CToggleBase::setEnabled(bool enabled)
+{
+	// for overrides
+}
+
 void CToggleBase::setSelected(bool on)
 {
 	bool changed = (on != selected);
@@ -363,6 +368,11 @@ void CToggleButton::doSelect(bool on)
 	}
 }
 
+void CToggleButton::setEnabled(bool enabled)
+{
+	setState(enabled ? NORMAL : BLOCKED);
+}
+
 void CToggleButton::clickLeft(tribool down, bool previousState)
 {
 	// force refresh
@@ -425,6 +435,17 @@ void CToggleGroup::setSelected(int id)
 	selectionChanged(id);
 }
 
+void CToggleGroup::setSelectedOnly(int id)
+{
+	for(auto it = buttons.begin(); it != buttons.end(); it++)
+	{
+		int buttonId = it->first;
+		buttons[buttonId]->setEnabled(buttonId == id);
+	}
+
+	selectionChanged(id);
+}
+
 void CToggleGroup::selectionChanged(int to)
 {
 	if (to == selectedID)
diff --git a/client/widgets/Buttons.h b/client/widgets/Buttons.h
index fe875c8e9..9e8ac298b 100644
--- a/client/widgets/Buttons.h
+++ b/client/widgets/Buttons.h
@@ -139,12 +139,17 @@ public:
 	void setSelected(bool on);
 
 	void addCallback(std::function<void(bool)> callback);
+
+	/// Set whether the toggle is currently enabled for user to use, this is only inplemented in ToggleButton, not for other toggles yet.
+	virtual void setEnabled(bool enabled);
 };
 
 /// A button which can be selected/deselected, checkbox
 class CToggleButton : public CButton, public CToggleBase
 {
 	void doSelect(bool on) override;
+	void setEnabled(bool enabled) override;
+
 public:
 	CToggleButton(Point position, const std::string &defName, const std::pair<std::string, std::string> &help,
 	              CFunctionList<void(bool)> Callback = 0, int key=0, bool playerColoredButton = false );
@@ -173,6 +178,9 @@ public:
 	void addToggle(int index, std::shared_ptr<CToggleBase> button);
 	/// Changes selection to specific value. Will select toggle with this ID, if present
 	void setSelected(int id);
+	/// in some cases, e.g. LoadGame difficulty selection, after refreshing the UI, the ToggleGroup should 
+	/// reset all of it's child buttons to BLOCK state, then make selection again
+	void setSelectedOnly(int id);
 };
 
 /// A typical slider for volume with an animated indicator