diff --git a/django/db/models/sql/compiler.py b/django/db/models/sql/compiler.py
index b70a15755b..7ada8e85d0 100644
--- a/django/db/models/sql/compiler.py
+++ b/django/db/models/sql/compiler.py
@@ -567,7 +567,7 @@ class SQLCompiler(object):
             # Firstly, avoid infinite loops.
             if not already_seen:
                 already_seen = set()
-            join_tuple = tuple(self.query.alias_map[j].table_name for j in joins)
+            join_tuple = tuple(getattr(self.query.alias_map[j], 'join_cols', None) for j in joins)
             if join_tuple in already_seen:
                 raise FieldError('Infinite loop caused by ordering.')
             already_seen.add(join_tuple)
diff --git a/tests/ordering/models.py b/tests/ordering/models.py
index 57a75027f6..c588a4e418 100644
--- a/tests/ordering/models.py
+++ b/tests/ordering/models.py
@@ -25,6 +25,7 @@ class Author(models.Model):
 @python_2_unicode_compatible
 class Article(models.Model):
     author = models.ForeignKey(Author, null=True)
+    second_author = models.ForeignKey(Author, null=True)
     headline = models.CharField(max_length=100)
     pub_date = models.DateTimeField()
 
@@ -33,3 +34,16 @@ class Article(models.Model):
 
     def __str__(self):
         return self.headline
+
+
+class OrderedByAuthorArticle(Article):
+    class Meta:
+        proxy = True
+        ordering = ('author', 'second_author')
+
+
+class Reference(models.Model):
+    article = models.ForeignKey(OrderedByAuthorArticle)
+
+    class Meta:
+        ordering = ('article',)
diff --git a/tests/ordering/tests.py b/tests/ordering/tests.py
index 19d289a2d1..027d5f1eb5 100644
--- a/tests/ordering/tests.py
+++ b/tests/ordering/tests.py
@@ -6,7 +6,7 @@ from operator import attrgetter
 from django.db.models import F
 from django.test import TestCase
 
-from .models import Article, Author
+from .models import Article, Author, Reference
 
 
 class OrderingTests(TestCase):
@@ -307,3 +307,20 @@ class OrderingTests(TestCase):
             ],
             attrgetter("headline")
         )
+
+    def test_related_ordering_duplicate_table_reference(self):
+        """
+        An ordering referencing a model with an ordering referencing a model
+        multiple time no circular reference should be detected (#24654).
+        """
+        first_author = Author.objects.create()
+        second_author = Author.objects.create()
+        self.a1.author = first_author
+        self.a1.second_author = second_author
+        self.a1.save()
+        self.a2.author = second_author
+        self.a2.second_author = first_author
+        self.a2.save()
+        r1 = Reference.objects.create(article_id=self.a1.pk)
+        r2 = Reference.objects.create(article_id=self.a2.pk)
+        self.assertQuerysetEqual(Reference.objects.all(), [r2, r1], lambda x: x)