diff --git a/changelog/5578.bugfix.rst b/changelog/5578.bugfix.rst
new file mode 100644
index 000000000..5f6c39185
--- /dev/null
+++ b/changelog/5578.bugfix.rst
@@ -0,0 +1,3 @@
+Improve type checking for some exception-raising functions (``pytest.xfail``, ``pytest.skip``, etc)
+so they provide better error messages when users meant to use marks (for example ``@pytest.xfail``
+instead of ``@pytest.mark.xfail``).
diff --git a/src/_pytest/outcomes.py b/src/_pytest/outcomes.py
index fb4d471b5..ca96a1b4f 100644
--- a/src/_pytest/outcomes.py
+++ b/src/_pytest/outcomes.py
@@ -13,16 +13,19 @@ class OutcomeException(BaseException):
     """
 
     def __init__(self, msg=None, pytrace=True):
+        if msg is not None and not isinstance(msg, str):
+            error_msg = (
+                "{} expected string as 'msg' parameter, got '{}' instead.\n"
+                "Perhaps you meant to use a mark?"
+            )
+            raise TypeError(error_msg.format(type(self).__name__, type(msg).__name__))
         BaseException.__init__(self, msg)
         self.msg = msg
         self.pytrace = pytrace
 
     def __repr__(self):
         if self.msg:
-            val = self.msg
-            if isinstance(val, bytes):
-                val = val.decode("UTF-8", errors="replace")
-            return val
+            return self.msg
         return "<{} instance>".format(self.__class__.__name__)
 
     __str__ = __repr__
diff --git a/testing/test_runner.py b/testing/test_runner.py
index 15180c071..82e413518 100644
--- a/testing/test_runner.py
+++ b/testing/test_runner.py
@@ -11,6 +11,7 @@ from _pytest import main
 from _pytest import outcomes
 from _pytest import reports
 from _pytest import runner
+from _pytest.outcomes import OutcomeException
 
 
 class TestSetupState:
@@ -990,3 +991,18 @@ class TestReportContents:
         rep = reports[1]
         assert rep.capstdout == ""
         assert rep.capstderr == ""
+
+
+def test_outcome_exception_bad_msg():
+    """Check that OutcomeExceptions validate their input to prevent confusing errors (#5578)"""
+
+    def func():
+        pass
+
+    expected = (
+        "OutcomeException expected string as 'msg' parameter, got 'function' instead.\n"
+        "Perhaps you meant to use a mark?"
+    )
+    with pytest.raises(TypeError) as excinfo:
+        OutcomeException(func)
+    assert str(excinfo.value) == expected
diff --git a/testing/test_skipping.py b/testing/test_skipping.py
index 6bb5f7aff..8bba479f1 100644
--- a/testing/test_skipping.py
+++ b/testing/test_skipping.py
@@ -1066,7 +1066,8 @@ def test_module_level_skip_error(testdir):
     testdir.makepyfile(
         """
         import pytest
-        @pytest.skip
+        pytest.skip("skip_module_level")
+
         def test_func():
             assert True
     """