diff --git a/pkg/commands/patch/patch_builder.go b/pkg/commands/patch/patch_builder.go
index 1edce0741..466b3da09 100644
--- a/pkg/commands/patch/patch_builder.go
+++ b/pkg/commands/patch/patch_builder.go
@@ -124,14 +124,6 @@ func (p *PatchBuilder) RemoveFile(filename string) error {
 	return nil
 }
 
-func getIndicesForRange(first, last int) []int {
-	indices := []int{}
-	for i := first; i <= last; i++ {
-		indices = append(indices, i)
-	}
-	return indices
-}
-
 func (p *PatchBuilder) getFileInfo(filename string) (*fileInfo, error) {
 	info, ok := p.fileInfoMap[filename]
 	if ok {
@@ -152,24 +144,24 @@ func (p *PatchBuilder) getFileInfo(filename string) (*fileInfo, error) {
 	return info, nil
 }
 
-func (p *PatchBuilder) AddFileLineRange(filename string, firstLineIdx, lastLineIdx int) error {
+func (p *PatchBuilder) AddFileLineRange(filename string, lineIndices []int) error {
 	info, err := p.getFileInfo(filename)
 	if err != nil {
 		return err
 	}
 	info.mode = PART
-	info.includedLineIndices = lo.Union(info.includedLineIndices, getIndicesForRange(firstLineIdx, lastLineIdx))
+	info.includedLineIndices = lo.Union(info.includedLineIndices, lineIndices)
 
 	return nil
 }
 
-func (p *PatchBuilder) RemoveFileLineRange(filename string, firstLineIdx, lastLineIdx int) error {
+func (p *PatchBuilder) RemoveFileLineRange(filename string, lineIndices []int) error {
 	info, err := p.getFileInfo(filename)
 	if err != nil {
 		return err
 	}
 	info.mode = PART
-	info.includedLineIndices, _ = lo.Difference(info.includedLineIndices, getIndicesForRange(firstLineIdx, lastLineIdx))
+	info.includedLineIndices, _ = lo.Difference(info.includedLineIndices, lineIndices)
 	if len(info.includedLineIndices) == 0 {
 		p.removeFile(info)
 	}
diff --git a/pkg/gui/controllers/patch_building_controller.go b/pkg/gui/controllers/patch_building_controller.go
index 94f987404..87328a15a 100644
--- a/pkg/gui/controllers/patch_building_controller.go
+++ b/pkg/gui/controllers/patch_building_controller.go
@@ -126,7 +126,6 @@ func (self *PatchBuildingController) toggleSelection() error {
 	self.context().GetMutex().Lock()
 	defer self.context().GetMutex().Unlock()
 
-	toggleFunc := self.c.Git().Patch.PatchBuilder.AddFileLineRange
 	filename := self.c.Contexts().CommitFiles.GetSelectedPath()
 	if filename == "" {
 		return nil
@@ -134,19 +133,26 @@ func (self *PatchBuildingController) toggleSelection() error {
 
 	state := self.context().GetState()
 
+	// Get added/deleted lines in the selected patch range
+	lineIndicesToToggle := state.ChangeLinesInSelectedPatchRange()
+	if len(lineIndicesToToggle) == 0 {
+		// Only context lines or header lines selected, so nothing to do
+		return nil
+	}
+
 	includedLineIndices, err := self.c.Git().Patch.PatchBuilder.GetFileIncLineIndices(filename)
 	if err != nil {
 		return err
 	}
-	currentLineIsStaged := lo.Contains(includedLineIndices, state.GetSelectedPatchLineIdx())
-	if currentLineIsStaged {
+
+	toggleFunc := self.c.Git().Patch.PatchBuilder.AddFileLineRange
+	firstSelectedChangeLineIsStaged := lo.Contains(includedLineIndices, lineIndicesToToggle[0])
+	if firstSelectedChangeLineIsStaged {
 		toggleFunc = self.c.Git().Patch.PatchBuilder.RemoveFileLineRange
 	}
 
 	// add range of lines to those set for the file
-	firstLineIdx, lastLineIdx := state.SelectedPatchRange()
-
-	if err := toggleFunc(filename, firstLineIdx, lastLineIdx); err != nil {
+	if err := toggleFunc(filename, lineIndicesToToggle); err != nil {
 		// might actually want to return an error here
 		self.c.Log.Error(err)
 	}
diff --git a/pkg/gui/patch_exploring/state.go b/pkg/gui/patch_exploring/state.go
index 53e120849..8c308b3d0 100644
--- a/pkg/gui/patch_exploring/state.go
+++ b/pkg/gui/patch_exploring/state.go
@@ -282,6 +282,20 @@ func (s *State) SelectedPatchRange() (int, int) {
 	return s.patchLineIndices[start], s.patchLineIndices[end]
 }
 
+// Returns the line indices of the selected patch range that are changes (i.e. additions or deletions)
+func (s *State) ChangeLinesInSelectedPatchRange() []int {
+	viewStart, viewEnd := s.SelectedViewRange()
+	patchStart, patchEnd := s.patchLineIndices[viewStart], s.patchLineIndices[viewEnd]
+	lines := s.patch.Lines()
+	indices := []int{}
+	for i := patchStart; i <= patchEnd; i++ {
+		if lines[i].IsChange() {
+			indices = append(indices, i)
+		}
+	}
+	return indices
+}
+
 func (s *State) CurrentLineNumber() int {
 	return s.patch.LineNumberOfLine(s.patchLineIndices[s.selectedLineIdx])
 }