diff --git a/asyncssh/config.py b/asyncssh/config.py
index 0376b74..1705b11 100644
--- a/asyncssh/config.py
+++ b/asyncssh/config.py
@@ -146,7 +146,7 @@ def _include(self, option: str, args: List[str]) -> None:
             else:
                 path = self._default_path
 
-            paths = list(path.glob(pattern))
+            paths = list(p for p in path.glob(pattern) if p.is_file())
 
             if not paths:
                 logger.debug1(f'Config pattern "{pattern}" matched no files')