修复SSH重载后私钥回连的短暂连接拒绝
This commit is contained in:
1 parent
1a317c2175
commit
714753236f
5 files changed
+146
-1
No files matched your search
@@ -1000,7 +1000,7 @@ func (s *Server) provisionServerSSHKey(ctx context.Context, item deployment.Serv
|
||||
}
|
||||
privateItem := item
|
||||
privateItem.AuthMethod = deployment.SSHAuthPrivateKey
|
||||
privateClient, err := s.connectSSHForKey(ctx, privateItem)
|
||||
privateClient, err := s.connectSSHAfterReload(ctx, privateItem)
|
||||
if err != nil {
|
||||
s.logSSHProvisionFailure("private_key_connect", item, err)
|
||||
item.SSHKeyStatus = deployment.SSHKeyStatusFailed
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
|
||||
)
|
||||
|
||||
func (s *Server) connectSSHAfterReload(ctx context.Context, item deployment.Server) (deployment.Client, error) {
|
||||
for attempt := 0; ; attempt++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client, err := s.connectSSHForKey(ctx, item)
|
||||
if err == nil {
|
||||
return client, nil
|
||||
}
|
||||
var networkError *net.OpError
|
||||
if attempt >= 4 || !errors.As(err, &networkError) || networkError.Op != "dial" || !errors.Is(err, syscall.ECONNREFUSED) {
|
||||
return nil, err
|
||||
}
|
||||
timer := time.NewTimer((100 * time.Millisecond) << attempt)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return nil, ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
|
||||
)
|
||||
|
||||
type reloadTestFactory struct {
|
||||
connect func(context.Context, deployment.Server) (deployment.Client, error)
|
||||
}
|
||||
|
||||
func (f reloadTestFactory) Connect(ctx context.Context, item deployment.Server) (deployment.Client, error) {
|
||||
return f.connect(ctx, item)
|
||||
}
|
||||
|
||||
type reloadTestClient struct{ deployment.Client }
|
||||
|
||||
func refusedReloadConnection() error {
|
||||
return fmt.Errorf("%w", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED})
|
||||
}
|
||||
|
||||
func TestSSHAfterReloadRetriesOnlyConnectionRefusal(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
err error
|
||||
failures int
|
||||
calls int
|
||||
}{
|
||||
{"ready", nil, 0, 1},
|
||||
{"recovered", refusedReloadConnection(), 2, 3},
|
||||
{"exhausted", refusedReloadConnection(), 5, 5},
|
||||
{"authentication", errors.New("authentication"), 1, 1},
|
||||
{"fingerprint", errors.New("fingerprint"), 1, 1},
|
||||
{"timeout", &net.OpError{Op: "dial", Err: syscall.ETIMEDOUT}, 1, 1},
|
||||
{"handshake", &net.OpError{Op: "read", Err: syscall.ECONNREFUSED}, 1, 1},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
calls := 0
|
||||
wantClient := &reloadTestClient{}
|
||||
server := &Server{deploymentFactory: reloadTestFactory{connect: func(ctx context.Context, item deployment.Server) (deployment.Client, error) {
|
||||
calls++
|
||||
if item.AuthMethod != deployment.SSHAuthPrivateKey || item.HostKeyFingerprint != "SHA256:registered" {
|
||||
t.Fatal(item.AuthMethod, item.HostKeyFingerprint)
|
||||
}
|
||||
if calls <= tc.failures {
|
||||
return nil, tc.err
|
||||
}
|
||||
return wantClient, nil
|
||||
}}}
|
||||
client, err := server.connectSSHAfterReload(t.Context(), deployment.Server{
|
||||
AuthMethod: deployment.SSHAuthPrivateKey, HostKeyFingerprint: "SHA256:registered",
|
||||
})
|
||||
if calls != tc.calls {
|
||||
t.Fatalf("calls=%d; want=%d", calls, tc.calls)
|
||||
}
|
||||
if tc.calls <= tc.failures {
|
||||
if !errors.Is(err, tc.err) || client != nil {
|
||||
t.Fatal(client, err)
|
||||
}
|
||||
} else if err != nil || client != wantClient {
|
||||
t.Fatal(client, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSSHAfterReloadHonorsCancellation(t *testing.T) {
|
||||
for _, cancelBeforeConnect := range []bool{false, true} {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
calls := 0
|
||||
server := &Server{deploymentFactory: reloadTestFactory{connect: func(context.Context, deployment.Server) (deployment.Client, error) {
|
||||
calls++
|
||||
cancel()
|
||||
return nil, refusedReloadConnection()
|
||||
}}}
|
||||
if cancelBeforeConnect {
|
||||
cancel()
|
||||
}
|
||||
client, err := server.connectSSHAfterReload(ctx, deployment.Server{})
|
||||
cancel()
|
||||
if !errors.Is(err, context.Canceled) || client != nil {
|
||||
t.Fatal(client, err)
|
||||
}
|
||||
wantCalls := 1
|
||||
if cancelBeforeConnect {
|
||||
wantCalls = 0
|
||||
}
|
||||
if calls != wantCalls {
|
||||
t.Fatalf("calls=%d; want=%d", calls, wantCalls)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user