diff --git a/AUTHORS b/AUTHORS
index 14882e754..84833c642 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -101,6 +101,7 @@ Lukas Bednar
 Luke Murphy
 Maciek Fijalkowski
 Maho
+Maik Figura
 Mandeep Bhutani
 Manuel Krebber
 Marc Schlaich
diff --git a/_pytest/python_api.py b/_pytest/python_api.py
index 051769398..457d30a10 100644
--- a/_pytest/python_api.py
+++ b/_pytest/python_api.py
@@ -7,6 +7,19 @@ from _pytest.compat import isclass, izip
 from _pytest.outcomes import fail
 import _pytest._code
 
+
+def _cmp_raises_type_error(self, other):
+    """__cmp__ implementation which raises TypeError. Used
+    by Approx base classes to implement only == and != and raise a
+    TypeError for other comparisons.
+
+    Needed in Python 2 only, Python 3 all it takes is not implementing the
+    other operators at all.
+    """
+    __tracebackhide__ = True
+    raise TypeError('Comparison operators other than == and != not supported by approx objects')
+
+
 # builtin pytest.approx helper
 
 
@@ -35,6 +48,9 @@ class ApproxBase(object):
     def __ne__(self, actual):
         return not (actual == self)
 
+    if sys.version_info[0] == 2:
+        __cmp__ = _cmp_raises_type_error
+
     def _approx_scalar(self, x):
         return ApproxScalar(x, rel=self.rel, abs=self.abs, nan_ok=self.nan_ok)
 
@@ -60,6 +76,9 @@ class ApproxNumpy(ApproxBase):
         return "approx({0!r})".format(list(
             self._approx_scalar(x) for x in self.expected))
 
+    if sys.version_info[0] == 2:
+        __cmp__ = _cmp_raises_type_error
+
     def __eq__(self, actual):
         import numpy as np
 
@@ -358,6 +377,24 @@ def approx(expected, rel=None, abs=None, nan_ok=False):
       is asymmetric and you can think of ``b`` as the reference value.  In the
       special case that you explicitly specify an absolute tolerance but not a
       relative tolerance, only the absolute tolerance is considered.
+
+    .. warning::
+
+       .. versionchanged:: 3.2
+
+       In order to avoid inconsistent behavior, ``TypeError`` is
+       raised for ``>``, ``>=``, ``<`` and ``<=`` comparisons.
+       The example below illustrates the problem::
+
+           assert approx(0.1) > 0.1 + 1e-10  # calls approx(0.1).__gt__(0.1 + 1e-10)
+           assert 0.1 + 1e-10 > approx(0.1)  # calls approx(0.1).__lt__(0.1 + 1e-10)
+
+       In the second example one expects ``approx(0.1).__le__(0.1 + 1e-10)``
+       to be called. But instead, ``approx(0.1).__lt__(0.1 + 1e-10)`` is used to
+       comparison. This is because the call hierarchy of rich comparisons
+       follows a fixed behavior. `More information...`__
+
+       __ https://docs.python.org/3/reference/datamodel.html#object.__ge__
     """
 
     from collections import Mapping, Sequence
diff --git a/changelog/2003.removal b/changelog/2003.removal
new file mode 100644
index 000000000..d3269bf4e
--- /dev/null
+++ b/changelog/2003.removal
@@ -0,0 +1,2 @@
+``pytest.approx`` no longer supports ``>``, ``>=``, ``<`` and ``<=`` operators to avoid surprising/inconsistent
+behavior. See `the docs <https://docs.pytest.org/en/latest/builtin.html#pytest.approx>`_ for more information.
diff --git a/testing/python/approx.py b/testing/python/approx.py
index 876226e06..d591b8ba5 100644
--- a/testing/python/approx.py
+++ b/testing/python/approx.py
@@ -1,4 +1,5 @@
 # encoding: utf-8
+import operator
 import sys
 import pytest
 import doctest
@@ -382,3 +383,16 @@ class TestApprox(object):
             '*At index 0 diff: 3 != 4 * {0}'.format(expected),
             '=* 1 failed in *=',
         ])
+
+    @pytest.mark.parametrize('op', [
+        pytest.param(operator.le, id='<='),
+        pytest.param(operator.lt, id='<'),
+        pytest.param(operator.ge, id='>='),
+        pytest.param(operator.gt, id='>'),
+    ])
+    def test_comparison_operator_type_error(self, op):
+        """
+        pytest.approx should raise TypeError for operators other than == and != (#2003).
+        """
+        with pytest.raises(TypeError):
+            op(1, approx(1, rel=1e-6, abs=1e-12))