diff --git a/CHANGELOG.md b/CHANGELOG.md
index ff7c727f..85460e2b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -36,7 +36,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - **MP4**: Renamed `AdvisoryRating::None` to `AdvisoryRating::Inoffensive`
 - Renamed `TaggedFile::remove_tag` to `TaggedFile::take`
 - **Vorbis Comments**: `VorbisComments::insert_picture` now accepts a user provided `PictureInformation`
-- **Vorbis Comments**: Rename `VorbisComments::{get_item, insert_item}` to `VorbisComments::{get, insert}`
+- **Vorbis Comments**: Rename `VorbisComments::{get_item, insert_item, remove_key}` to `VorbisComments::{get, insert, remove}`
+- **Vorbis Comments**: `VorbisComments::remove` now returns an iterator over the removed items
 
 ### Fixed
 - **MP4**: Non-full `meta` atoms are now properly handled.
diff --git a/src/ogg/tag.rs b/src/ogg/tag.rs
index ba27cba3..cb665246 100644
--- a/src/ogg/tag.rs
+++ b/src/ogg/tag.rs
@@ -26,7 +26,7 @@ macro_rules! impl_accessor {
 					}
 
 					fn [<remove_ $name>](&mut self) {
-						self.remove_key($key)
+						let _ = self.remove($key);
 					}
 				)+
 			}
@@ -92,11 +92,21 @@ impl VorbisComments {
 		self.items.push((key, value))
 	}
 
-	/// Removes an item by key
+	/// Removes all items with a key, returning an iterator
 	///
 	/// NOTE: This is case-sensitive
-	pub fn remove_key(&mut self, key: &str) {
-		self.items.retain(|(k, _)| k != key);
+	pub fn remove(&mut self, key: &str) -> impl Iterator<Item = String> + '_ {
+		// TODO: drain_filter
+		let mut split_idx = 0_usize;
+
+		for read_idx in 0..self.items.len() {
+			if self.items[read_idx].0 == key {
+				self.items.swap(split_idx, read_idx);
+				split_idx += 1;
+			}
+		}
+
+		self.items.drain(..split_idx).map(|(_, v)| v)
 	}
 
 	/// Inserts a [`Picture`]