diff --git a/src/_pytest/main.py b/src/_pytest/main.py
index 084d68dab..ed4204b08 100644
--- a/src/_pytest/main.py
+++ b/src/_pytest/main.py
@@ -476,7 +476,6 @@ class Session(nodes.FSCollector):
             for arg, exc in self._notfound:
                 line = "(no name {!r} in any of {!r})".format(arg, exc.args[0])
                 errors.append("not found: {}\n{}".format(arg, line))
-                # XXX: test this
             raise UsageError(*errors)
         if not genitems:
             return rep.result
diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py
index 82c727fc6..578ab45eb 100644
--- a/testing/acceptance_test.py
+++ b/testing/acceptance_test.py
@@ -178,8 +178,14 @@ class TestGeneralUsage:
         p1 = testdir.makepyfile("")
         p2 = testdir.makefile(".pyc", "123")
         result = testdir.runpytest(p1, p2)
-        assert result.ret
-        result.stderr.fnmatch_lines(["*ERROR: not found:*{}".format(p2.basename)])
+        assert result.ret == ExitCode.USAGE_ERROR
+        result.stderr.fnmatch_lines(
+            [
+                "ERROR: not found: {}".format(p2),
+                "(no name {!r} in any of [[][]])".format(str(p2)),
+                "",
+            ]
+        )
 
     @pytest.mark.filterwarnings("default")
     def test_better_reporting_on_conftest_load_failure(self, testdir, request):