diff --git a/django/contrib/admin/options.py b/django/contrib/admin/options.py
index 0b7a6ae7d97..a0b816ccf00 100644
--- a/django/contrib/admin/options.py
+++ b/django/contrib/admin/options.py
@@ -339,6 +339,11 @@ class BaseModelAdmin(six.with_metaclass(RenameBaseModelAdminMethods)):
         except FieldDoesNotExist:
             return False
 
+        # Check whether this model is the origin of a M2M relationship
+        # in which case to_field has to be the pk on this model.
+        if opts.many_to_many and field.primary_key:
+            return True
+
         # Make sure at least one of the models registered for this site
         # references this field through a FK or a M2M relationship.
         registered_models = set()
diff --git a/docs/releases/1.4.16.txt b/docs/releases/1.4.16.txt
index b10a72d2641..23e9ecd380f 100644
--- a/docs/releases/1.4.16.txt
+++ b/docs/releases/1.4.16.txt
@@ -4,12 +4,15 @@ Django 1.4.16 release notes
 
 *Under development*
 
-Django 1.4.16 fixes a regression in the 1.4.14 security release and a bug
-preventing the use of some GEOS versions with GeoDjango.
+Django 1.4.16 fixes a couple regressions in the 1.4.14 security release and a
+bug preventing the use of some GEOS versions with GeoDjango.
 
 Bugfixes
 ========
 
+* Allowed related many-to-many fields to be referenced in the admin
+  (`#23604 <http://code.djangoproject.com/ticket/23604>`_).
+
 * Allowed inline and hidden references to admin fields
   (`#23431 <http://code.djangoproject.com/ticket/23431>`_).
 
diff --git a/docs/releases/1.5.11.txt b/docs/releases/1.5.11.txt
index 9a60239c645..3c6f4c42ac2 100644
--- a/docs/releases/1.5.11.txt
+++ b/docs/releases/1.5.11.txt
@@ -4,10 +4,13 @@ Django 1.5.11 release notes
 
 *Under development*
 
-Django 1.5.11 fixes a regression in the 1.5.9 security release.
+Django 1.5.11 fixes a couple regressions in the 1.5.9 security release.
 
 Bugfixes
 ========
 
+* Allowed related many-to-many fields to be referenced in the admin
+  (`#23604 <http://code.djangoproject.com/ticket/23604>`_).
+
 * Allowed inline and hidden references to admin fields
   (`#23431 <http://code.djangoproject.com/ticket/23431>`_).
diff --git a/docs/releases/1.6.8.txt b/docs/releases/1.6.8.txt
index b209649ba4a..9f8d3ab3a5b 100644
--- a/docs/releases/1.6.8.txt
+++ b/docs/releases/1.6.8.txt
@@ -4,9 +4,12 @@ Django 1.6.8 release notes
 
 *Under development*
 
-Django 1.6.8 fixes a regression in the 1.6.6 security release.
+Django 1.6.8 fixes a couple regressions in the 1.6.6 security release.
 
 Bugfixes
 ========
 
+* Allowed related many-to-many fields to be referenced in the admin
+  (:ticket:`23604`).
+
 * Allowed inline and hidden references to admin fields (:ticket:`23431`).
diff --git a/tests/admin_views/admin.py b/tests/admin_views/admin.py
index 62ea8f889be..ca8a9fdbf68 100644
--- a/tests/admin_views/admin.py
+++ b/tests/admin_views/admin.py
@@ -31,7 +31,7 @@ from .models import (Article, Chapter, Account, Media, Child, Parent, Picture,
     AdminOrderedCallable, Report, Color2, UnorderedObject, MainPrepopulated,
     RelatedPrepopulated, UndeletableObject, UserMessenger, Simple, Choice,
     ShortMessage, Telegram, ReferencedByParent, ChildOfReferer, M2MReference,
-    ReferencedByInline, InlineReference, InlineReferer)
+    ReferencedByInline, InlineReference, InlineReferer, Ingredient)
 
 
 def callable_year(dt_value):
@@ -789,6 +789,7 @@ site.register(Color2, CustomTemplateFilterColorAdmin)
 site.register(Simple, AttributeErrorRaisingAdmin)
 site.register(UserMessenger, MessageTestingAdmin)
 site.register(Choice, ChoiceList)
+site.register(Ingredient)
 
 # Register core models we need in our tests
 from django.contrib.auth.models import User, Group
diff --git a/tests/admin_views/models.py b/tests/admin_views/models.py
index 91b4aa7f1d3..7b8af6f8d2c 100644
--- a/tests/admin_views/models.py
+++ b/tests/admin_views/models.py
@@ -716,3 +716,12 @@ class InlineReference(models.Model):
 class InlineReferer(models.Model):
     refs = models.ManyToManyField(InlineReference)
 
+
+# Models for #23604
+class Recipe(models.Model):
+    name = models.CharField(max_length=20)
+
+
+class Ingredient(models.Model):
+    name = models.CharField(max_length=20)
+    recipes = models.ManyToManyField('Recipe', related_name='ingredients')
diff --git a/tests/admin_views/tests.py b/tests/admin_views/tests.py
index 01a2288739e..3913baa63fc 100644
--- a/tests/admin_views/tests.py
+++ b/tests/admin_views/tests.py
@@ -599,6 +599,10 @@ class AdminViewBasicTest(AdminViewBasicTestCase):
         response = self.client.get("/test_admin/admin/admin_views/m2mreference/", {TO_FIELD_VAR: 'id'})
         self.assertEqual(response.status_code, 200)
 
+        # #23604 - Specifying the pk of this model should be allowed when this model defines a m2m relationship
+        response = self.client.get("/test_admin/admin/admin_views/ingredient/", {TO_FIELD_VAR: 'id'})
+        self.assertEqual(response.status_code, 200)
+
         # #23329 - Specifying a field that is not refered by any other model directly registered
         # to this admin site but registered through inheritance should be allowed.
         response = self.client.get("/test_admin/admin/admin_views/referencedbyparent/", {TO_FIELD_VAR: 'id'})