diff --git a/pcsx2/Input/SDLInputSource.cpp b/pcsx2/Input/SDLInputSource.cpp
index c9012196ffeb5..7a0a8cf77ef53 100644
--- a/pcsx2/Input/SDLInputSource.cpp
+++ b/pcsx2/Input/SDLInputSource.cpp
@@ -247,6 +247,14 @@ u32 SDLInputSource::ParseRGBForPlayerId(const std::string_view str, u32 player_i
 	return color;
 }
 
+void SDLInputSource::ResetRGBForAllPlayers(SettingsInterface& si)
+{
+	for (u32 player_id = 0; player_id < MAX_LED_COLORS; player_id++)
+	{
+		si.DeleteValue("SDLExtra", fmt::format("Player{}LED", player_id).c_str());
+	}
+}
+
 void SDLInputSource::SetHints()
 {
 	if (const std::string upath = Path::Combine(EmuFolders::DataRoot, CONTROLLER_DB_FILENAME); FileSystem::FileExists(upath.c_str()))
diff --git a/pcsx2/Input/SDLInputSource.h b/pcsx2/Input/SDLInputSource.h
index 5c10b1699167f..5593c88918c20 100644
--- a/pcsx2/Input/SDLInputSource.h
+++ b/pcsx2/Input/SDLInputSource.h
@@ -44,6 +44,7 @@ class SDLInputSource final : public InputSource
 
 	static u32 GetRGBForPlayerId(SettingsInterface& si, u32 player_id);
 	static u32 ParseRGBForPlayerId(const std::string_view str, u32 player_id);
+	static void ResetRGBForAllPlayers(SettingsInterface& si);
 
 private:
 	struct ControllerData
diff --git a/pcsx2/SIO/Pad/Pad.cpp b/pcsx2/SIO/Pad/Pad.cpp
index ae284f0bc2b6e..a0fd1ceedd60d 100644
--- a/pcsx2/SIO/Pad/Pad.cpp
+++ b/pcsx2/SIO/Pad/Pad.cpp
@@ -12,6 +12,8 @@
 #include "SIO/Pad/PadNotConnected.h"
 #include "SIO/Sio.h"
 
+#include "Input/SDLInputSource.h"
+
 #include "IconsFontAwesome5.h"
 
 #include "VMManager.h"
@@ -167,6 +169,7 @@ void Pad::SetDefaultControllerConfig(SettingsInterface& si)
 	si.SetBoolValue("Pad", "MultitapPort2", false);
 	si.SetFloatValue("Pad", "PointerXScale", 8.0f);
 	si.SetFloatValue("Pad", "PointerYScale", 8.0f);
+	SDLInputSource::ResetRGBForAllPlayers(si);
 
 	// PCSX2 Controller Settings - Default pad types and parameters.
 	for (u32 i = 0; i < Pad::NUM_CONTROLLER_PORTS; i++)