diff --git a/pam_kwallet.c b/pam_kwallet.c
--- a/pam_kwallet.c
+++ b/pam_kwallet.c
@@ -72,6 +72,7 @@
 const static char *socketPath = NULL;
 const static char *kwalletPamDataKey = NULL;
 const static char *logPrefix = NULL;
+static int force_run = 0;
 
 #ifdef KWALLET5
 const static char *envVar = "PAM_KWALLET5_LOGIN";
@@ -98,6 +99,8 @@
             kwalletd = argv[x] + 9;
         } else if (strstr(argv[x], "socketPath=") != NULL) {
             socketPath= argv[x] + 11;
+        } else if (strcmp(argv[x], "force_run") == 0) {
+            force_run = 1;
         }
     }
 #ifdef KWALLET5
@@ -240,6 +243,24 @@
     return result;
 }
 
+static int is_graphical_session(pam_handle_t *pamh)
+{
+    //Detect a graphical session
+    const char *pam_tty = NULL, *pam_xdisplay = NULL,
+               *xdg_session_type = NULL, *display = NULL;
+
+    pam_get_item(pamh, PAM_TTY, (const void**) &pam_tty);
+#ifdef PAM_XDISPLAY
+    pam_get_item(pamh, PAM_XDISPLAY, (const void**) &pam_xdisplay);
+#endif
+    xdg_session_type = get_env(pamh, "XDG_SESSION_TYPE");
+
+    return (pam_xdisplay && strlen(pam_xdisplay) != 0)
+           || (pam_tty && pam_tty[0] == ':')
+           || (xdg_session_type && strcmp(xdg_session_type, "x11") == 0)
+           || (xdg_session_type && strcmp(xdg_session_type, "wayland") == 0);
+}
+
 PAM_EXTERN int pam_sm_authenticate(pam_handle_t *pamh, int flags, int argc, const char **argv)
 {
     pam_syslog(pamh, LOG_INFO, "%s: pam_sm_authenticate\n", logPrefix);
@@ -525,6 +546,11 @@
 
     parseArguments(argc, argv);
 
+    if (!force_run && !is_graphical_session(pamh)) {
+        pam_syslog(pamh, LOG_INFO, "%s: not a graphical session, skipping. Use force_run parameter to ignore this.", logPrefix);
+        return PAM_IGNORE;
+    }
+
     int result;
     result = pam_set_data(pamh, "sm_open_session", "1", NULL);
     if (result != PAM_SUCCESS) {