diff --git a/pkg/bindings/connection.go b/pkg/bindings/connection.go
index 0fce6c9a27..abf7ab496e 100644
--- a/pkg/bindings/connection.go
+++ b/pkg/bindings/connection.go
@@ -11,6 +11,7 @@ import (
 	"net/url"
 	"os"
 	"os/user"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -147,59 +148,87 @@ func NewConnectionWithIdentity(ctx context.Context, uri string, identity string,
 
 func sshClient(_url *url.URL, uri string, identity string, machine bool) (Connection, error) {
 	var (
-		err error
+		err  error
+		port int
 	)
 	connection := Connection{
 		URI: _url,
 	}
 	userinfo := _url.User
-	if _url.User == nil {
-		u, err := user.Current()
-		if err != nil {
-			return connection, fmt.Errorf("current user could not be determined: %w", err)
-		}
-		userinfo = url.User(u.Username)
-	}
-	port := 22
+
 	if _url.Port() != "" {
 		port, err = strconv.Atoi(_url.Port())
 		if err != nil {
 			return connection, err
 		}
 	}
-	// ssh_config
-	alias := _url.Hostname()
-	cfg := ssh_config.DefaultUserSettings
-	found := false
-	if val := cfg.Get(alias, "User"); val != "" {
-		userinfo = url.User(val)
-		found = true
-	}
-	if val := cfg.Get(alias, "Hostname"); val != "" {
-		uri = val
-		found = true
-	}
-	if val := cfg.Get(alias, "Port"); val != "" {
-		if val != ssh_config.Default("Port") {
-			port, err = strconv.Atoi(val)
+
+	// only parse ssh_config when we are not connecting to a machine
+	// For machine connections we always have the full URL in the
+	// system connection so reading the file is just unnecessary.
+	if !machine {
+		alias := _url.Hostname()
+		cfg := ssh_config.DefaultUserSettings
+		cfg.IgnoreErrors = true
+		found := false
+
+		if userinfo == nil {
+			if val := cfg.Get(alias, "User"); val != "" {
+				userinfo = url.User(val)
+				found = true
+			}
+		}
+		// not in url or ssh_config so default to current user
+		if userinfo == nil {
+			u, err := user.Current()
 			if err != nil {
-				return connection, fmt.Errorf("port is not an int: %s: %w", val, err)
+				return connection, fmt.Errorf("current user could not be determined: %w", err)
 			}
-			found = true
+			userinfo = url.User(u.Username)
 		}
-	}
-	if val := cfg.Get(alias, "IdentityFile"); val != "" {
-		if val != ssh_config.Default("IdentityFile") {
-			identity = strings.Trim(val, "\"")
+
+		if val := cfg.Get(alias, "Hostname"); val != "" {
+			uri = val
 			found = true
 		}
-	}
-	if found {
-		logrus.Debugf("ssh_config alias found: %s", alias)
-		logrus.Debugf("  User: %s", userinfo.Username())
-		logrus.Debugf("  Hostname: %s", uri)
-		logrus.Debugf("  Port: %d", port)
-		logrus.Debugf("  IdentityFile: %q", identity)
+
+		if port == 0 {
+			if val := cfg.Get(alias, "Port"); val != "" {
+				if val != ssh_config.Default("Port") {
+					port, err = strconv.Atoi(val)
+					if err != nil {
+						return connection, fmt.Errorf("port is not an int: %s: %w", val, err)
+					}
+					found = true
+				}
+			}
+		}
+		// not in ssh config or url so use default 22 port
+		if port == 0 {
+			port = 22
+		}
+
+		if identity == "" {
+			if val := cfg.Get(alias, "IdentityFile"); val != "" {
+				identity = strings.Trim(val, "\"")
+				if strings.HasPrefix(identity, "~/") {
+					homedir, err := os.UserHomeDir()
+					if err != nil {
+						return connection, fmt.Errorf("failed to find home dir: %w", err)
+					}
+					identity = filepath.Join(homedir, identity[2:])
+				}
+				found = true
+			}
+		}
+
+		if found {
+			logrus.Debugf("ssh_config alias found: %s", alias)
+			logrus.Debugf("  User: %s", userinfo.Username())
+			logrus.Debugf("  Hostname: %s", uri)
+			logrus.Debugf("  Port: %d", port)
+			logrus.Debugf("  IdentityFile: %q", identity)
+		}
 	}
 	conn, err := ssh.Dial(&ssh.ConnectionDialOptions{
 		Host:                        uri,
diff --git a/pkg/machine/e2e/machine_test.go b/pkg/machine/e2e/machine_test.go
index ee1d57c3d1..7ef0c60549 100644
--- a/pkg/machine/e2e/machine_test.go
+++ b/pkg/machine/e2e/machine_test.go
@@ -98,6 +98,19 @@ var _ = SynchronizedAfterSuite(func() {}, func() {
 	}
 })
 
+// The config does not matter to much for our testing, however we
+// would like to be sure podman machine is not effected by certain
+// settings as we should be using full URLs anywhere.
+// https://github.com/containers/podman/issues/24567
+const sshConfigContent = `
+Host *
+  User NOT_REAL
+  Port 9999
+Host 127.0.0.1
+  User blah
+  IdentityFile ~/.ssh/id_ed25519
+`
+
 func setup() (string, *machineTestBuilder) {
 	// Set TMPDIR if this needs a new directory
 	if value, ok := os.LookupEnv("TMPDIR"); ok {
@@ -118,7 +131,7 @@ func setup() (string, *machineTestBuilder) {
 	if err != nil {
 		Fail(fmt.Sprintf("failed to create ssh config: %q", err))
 	}
-	if _, err := sshConfig.WriteString("IdentitiesOnly=yes"); err != nil {
+	if _, err := sshConfig.WriteString(sshConfigContent); err != nil {
 		Fail(fmt.Sprintf("failed to write ssh config: %q", err))
 	}
 	if err := sshConfig.Close(); err != nil {