diff --git a/django/contrib/admin/options.py b/django/contrib/admin/options.py
index 0bf9ede6cb..12291605ee 100644
--- a/django/contrib/admin/options.py
+++ b/django/contrib/admin/options.py
@@ -281,6 +281,11 @@ class BaseModelAdmin(object):
         except FieldDoesNotExist:
             return False
 
+        # Check whether this model is the origin of a M2M relationship
+        # in which case to_field has to be the pk on this model.
+        if opts.many_to_many and field.primary_key:
+            return True
+
         # Make sure at least one of the models registered for this site
         # references this field through a FK or a M2M relationship.
         registered_models = set()
diff --git a/docs/releases/1.4.16.txt b/docs/releases/1.4.16.txt
index b10a72d264..23e9ecd380 100644
--- a/docs/releases/1.4.16.txt
+++ b/docs/releases/1.4.16.txt
@@ -4,12 +4,15 @@ Django 1.4.16 release notes
 
 *Under development*
 
-Django 1.4.16 fixes a regression in the 1.4.14 security release and a bug
-preventing the use of some GEOS versions with GeoDjango.
+Django 1.4.16 fixes a couple regressions in the 1.4.14 security release and a
+bug preventing the use of some GEOS versions with GeoDjango.
 
 Bugfixes
 ========
 
+* Allowed related many-to-many fields to be referenced in the admin
+  (`#23604 <http://code.djangoproject.com/ticket/23604>`_).
+
 * Allowed inline and hidden references to admin fields
   (`#23431 <http://code.djangoproject.com/ticket/23431>`_).
 
diff --git a/tests/regressiontests/admin_views/admin.py b/tests/regressiontests/admin_views/admin.py
index f1147f6c47..d6c26ac535 100644
--- a/tests/regressiontests/admin_views/admin.py
+++ b/tests/regressiontests/admin_views/admin.py
@@ -28,7 +28,7 @@ from .models import (Article, Chapter, Account, Media, Child, Parent, Picture,
     AdminOrderedField, AdminOrderedModelMethod, AdminOrderedAdminMethod,
     AdminOrderedCallable, Report, Color2, UnorderedObject, MainPrepopulated,
     RelatedPrepopulated, ReferencedByParent, ChildOfReferer, M2MReference,
-    ReferencedByInline, InlineReference, InlineReferer)
+    ReferencedByInline, InlineReference, InlineReferer, Ingredient)
 
 
 def callable_year(dt_value):
@@ -656,6 +656,7 @@ site.register(AdminOrderedModelMethod, AdminOrderedModelMethodAdmin)
 site.register(AdminOrderedAdminMethod, AdminOrderedAdminMethodAdmin)
 site.register(AdminOrderedCallable, AdminOrderedCallableAdmin)
 site.register(Color2, CustomTemplateFilterColorAdmin)
+site.register(Ingredient)
 
 # Register core models we need in our tests
 from django.contrib.auth.models import User, Group
diff --git a/tests/regressiontests/admin_views/models.py b/tests/regressiontests/admin_views/models.py
index 7378dcbe96..5546a09e64 100644
--- a/tests/regressiontests/admin_views/models.py
+++ b/tests/regressiontests/admin_views/models.py
@@ -637,3 +637,12 @@ class InlineReference(models.Model):
 class InlineReferer(models.Model):
     refs = models.ManyToManyField(InlineReference)
 
+
+# Models for #23604
+class Recipe(models.Model):
+    name = models.CharField(max_length=20)
+
+
+class Ingredient(models.Model):
+    name = models.CharField(max_length=20)
+    recipes = models.ManyToManyField('Recipe', related_name='ingredients')
diff --git a/tests/regressiontests/admin_views/tests.py b/tests/regressiontests/admin_views/tests.py
index eb2caf8878..4a972d215d 100644
--- a/tests/regressiontests/admin_views/tests.py
+++ b/tests/regressiontests/admin_views/tests.py
@@ -590,6 +590,10 @@ class AdminViewBasicTest(TestCase):
         response = self.client.get("/test_admin/admin/admin_views/m2mreference/", {TO_FIELD_VAR: 'id'})
         self.assertEqual(response.status_code, 200)
 
+        # #23604 - Specifying the pk of this model should be allowed when this model defines a m2m relationship
+        response = self.client.get("/test_admin/admin/admin_views/ingredient/", {TO_FIELD_VAR: 'id'})
+        self.assertEqual(response.status_code, 200)
+
         # #23329 - Specifying a field that is not refered by any other model directly registered
         # to this admin site but registered through inheritance should be allowed.
         response = self.client.get("/test_admin/admin/admin_views/referencedbyparent/", {TO_FIELD_VAR: 'id'})