diff --git a/pkg/gui/controllers/patch_building_controller.go b/pkg/gui/controllers/patch_building_controller.go
index 173ee979a..166cc4a2b 100644
--- a/pkg/gui/controllers/patch_building_controller.go
+++ b/pkg/gui/controllers/patch_building_controller.go
@@ -170,7 +170,7 @@ func (self *PatchBuildingController) Escape() error {
 	context := self.c.Contexts().CustomPatchBuilder
 	state := context.GetState()
 
-	if state.SelectingRange() || state.SelectingHunk() {
+	if state.SelectingRange() || state.SelectingHunkEnabledByUser() {
 		state.SetLineSelectMode()
 		self.c.PostRefreshUpdate(context)
 		return nil
diff --git a/pkg/gui/controllers/staging_controller.go b/pkg/gui/controllers/staging_controller.go
index 6b251883b..44bcffb19 100644
--- a/pkg/gui/controllers/staging_controller.go
+++ b/pkg/gui/controllers/staging_controller.go
@@ -168,7 +168,7 @@ func (self *StagingController) EditFile() error {
 }
 
 func (self *StagingController) Escape() error {
-	if self.context.GetState().SelectingRange() || self.context.GetState().SelectingHunk() {
+	if self.context.GetState().SelectingRange() || self.context.GetState().SelectingHunkEnabledByUser() {
 		self.context.GetState().SetLineSelectMode()
 		self.c.PostRefreshUpdate(self.context)
 		return nil
diff --git a/pkg/gui/patch_exploring/state.go b/pkg/gui/patch_exploring/state.go
index 01d34b4e0..6223a9979 100644
--- a/pkg/gui/patch_exploring/state.go
+++ b/pkg/gui/patch_exploring/state.go
@@ -28,6 +28,12 @@ type State struct {
 	viewLineIndices []int
 	// Array of indices of the original patch lines indexed by a wrapped view line index
 	patchLineIndices []int
+
+	// whether the user has switched to hunk mode manually; if hunk mode is on
+	// but this is false, then hunk mode was enabled because the config makes it
+	// on by default.
+	// this makes a difference for whether we want to escape out of hunk mode
+	userEnabledHunkMode bool
 }
 
 // these represent what select mode we're in
@@ -65,6 +71,11 @@ func NewState(diff string, selectedLineIdx int, view *gocui.View, oldState *Stat
 		selectMode = HUNK
 	}
 
+	userEnabledHunkMode := false
+	if oldState != nil {
+		userEnabledHunkMode = oldState.userEnabledHunkMode
+	}
+
 	// if we have clicked from the outside to focus the main view we'll pass in a non-negative line index so that we can instantly select that line
 	if selectedLineIdx >= 0 {
 		// Clamp to the number of wrapped view lines; index might be out of
@@ -84,14 +95,15 @@ func NewState(diff string, selectedLineIdx int, view *gocui.View, oldState *Stat
 	}
 
 	return &State{
-		patch:             patch,
-		selectedLineIdx:   selectedLineIdx,
-		selectMode:        selectMode,
-		rangeStartLineIdx: rangeStartLineIdx,
-		rangeIsSticky:     false,
-		diff:              diff,
-		viewLineIndices:   viewLineIndices,
-		patchLineIndices:  patchLineIndices,
+		patch:               patch,
+		selectedLineIdx:     selectedLineIdx,
+		selectMode:          selectMode,
+		rangeStartLineIdx:   rangeStartLineIdx,
+		rangeIsSticky:       false,
+		diff:                diff,
+		viewLineIndices:     viewLineIndices,
+		patchLineIndices:    patchLineIndices,
+		userEnabledHunkMode: userEnabledHunkMode,
 	}
 }
 
@@ -129,6 +141,7 @@ func (s *State) ToggleSelectHunk() {
 		s.selectMode = LINE
 	} else {
 		s.selectMode = HUNK
+		s.userEnabledHunkMode = true
 
 		// If we are not currently on a change line, select the next one (or the
 		// previous one if there is no next one):
@@ -159,6 +172,10 @@ func (s *State) SelectingHunk() bool {
 	return s.selectMode == HUNK
 }
 
+func (s *State) SelectingHunkEnabledByUser() bool {
+	return s.selectMode == HUNK && s.userEnabledHunkMode
+}
+
 func (s *State) SelectingRange() bool {
 	return s.selectMode == RANGE && (s.rangeIsSticky || s.rangeStartLineIdx != s.selectedLineIdx)
 }