修复SSH重载后私钥回连的短暂连接拒绝
This commit is contained in:
parent
1a317c2175
commit
714753236f
@ -1000,7 +1000,7 @@ func (s *Server) provisionServerSSHKey(ctx context.Context, item deployment.Serv
|
||||
}
|
||||
privateItem := item
|
||||
privateItem.AuthMethod = deployment.SSHAuthPrivateKey
|
||||
privateClient, err := s.connectSSHForKey(ctx, privateItem)
|
||||
privateClient, err := s.connectSSHAfterReload(ctx, privateItem)
|
||||
if err != nil {
|
||||
s.logSSHProvisionFailure("private_key_connect", item, err)
|
||||
item.SSHKeyStatus = deployment.SSHKeyStatusFailed
|
||||
|
||||
34
backend/internal/httpapi/ssh_reload.go
Normal file
34
backend/internal/httpapi/ssh_reload.go
Normal file
@ -0,0 +1,34 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
|
||||
)
|
||||
|
||||
func (s *Server) connectSSHAfterReload(ctx context.Context, item deployment.Server) (deployment.Client, error) {
|
||||
for attempt := 0; ; attempt++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client, err := s.connectSSHForKey(ctx, item)
|
||||
if err == nil {
|
||||
return client, nil
|
||||
}
|
||||
var networkError *net.OpError
|
||||
if attempt >= 4 || !errors.As(err, &networkError) || networkError.Op != "dial" || !errors.Is(err, syscall.ECONNREFUSED) {
|
||||
return nil, err
|
||||
}
|
||||
timer := time.NewTimer((100 * time.Millisecond) << attempt)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return nil, ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
98
backend/internal/httpapi/ssh_reload_test.go
Normal file
98
backend/internal/httpapi/ssh_reload_test.go
Normal file
@ -0,0 +1,98 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
|
||||
)
|
||||
|
||||
type reloadTestFactory struct {
|
||||
connect func(context.Context, deployment.Server) (deployment.Client, error)
|
||||
}
|
||||
|
||||
func (f reloadTestFactory) Connect(ctx context.Context, item deployment.Server) (deployment.Client, error) {
|
||||
return f.connect(ctx, item)
|
||||
}
|
||||
|
||||
type reloadTestClient struct{ deployment.Client }
|
||||
|
||||
func refusedReloadConnection() error {
|
||||
return fmt.Errorf("%w", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED})
|
||||
}
|
||||
|
||||
func TestSSHAfterReloadRetriesOnlyConnectionRefusal(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
err error
|
||||
failures int
|
||||
calls int
|
||||
}{
|
||||
{"ready", nil, 0, 1},
|
||||
{"recovered", refusedReloadConnection(), 2, 3},
|
||||
{"exhausted", refusedReloadConnection(), 5, 5},
|
||||
{"authentication", errors.New("authentication"), 1, 1},
|
||||
{"fingerprint", errors.New("fingerprint"), 1, 1},
|
||||
{"timeout", &net.OpError{Op: "dial", Err: syscall.ETIMEDOUT}, 1, 1},
|
||||
{"handshake", &net.OpError{Op: "read", Err: syscall.ECONNREFUSED}, 1, 1},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
calls := 0
|
||||
wantClient := &reloadTestClient{}
|
||||
server := &Server{deploymentFactory: reloadTestFactory{connect: func(ctx context.Context, item deployment.Server) (deployment.Client, error) {
|
||||
calls++
|
||||
if item.AuthMethod != deployment.SSHAuthPrivateKey || item.HostKeyFingerprint != "SHA256:registered" {
|
||||
t.Fatal(item.AuthMethod, item.HostKeyFingerprint)
|
||||
}
|
||||
if calls <= tc.failures {
|
||||
return nil, tc.err
|
||||
}
|
||||
return wantClient, nil
|
||||
}}}
|
||||
client, err := server.connectSSHAfterReload(t.Context(), deployment.Server{
|
||||
AuthMethod: deployment.SSHAuthPrivateKey, HostKeyFingerprint: "SHA256:registered",
|
||||
})
|
||||
if calls != tc.calls {
|
||||
t.Fatalf("calls=%d; want=%d", calls, tc.calls)
|
||||
}
|
||||
if tc.calls <= tc.failures {
|
||||
if !errors.Is(err, tc.err) || client != nil {
|
||||
t.Fatal(client, err)
|
||||
}
|
||||
} else if err != nil || client != wantClient {
|
||||
t.Fatal(client, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSSHAfterReloadHonorsCancellation(t *testing.T) {
|
||||
for _, cancelBeforeConnect := range []bool{false, true} {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
calls := 0
|
||||
server := &Server{deploymentFactory: reloadTestFactory{connect: func(context.Context, deployment.Server) (deployment.Client, error) {
|
||||
calls++
|
||||
cancel()
|
||||
return nil, refusedReloadConnection()
|
||||
}}}
|
||||
if cancelBeforeConnect {
|
||||
cancel()
|
||||
}
|
||||
client, err := server.connectSSHAfterReload(ctx, deployment.Server{})
|
||||
cancel()
|
||||
if !errors.Is(err, context.Canceled) || client != nil {
|
||||
t.Fatal(client, err)
|
||||
}
|
||||
wantCalls := 1
|
||||
if cancelBeforeConnect {
|
||||
wantCalls = 0
|
||||
}
|
||||
if calls != wantCalls {
|
||||
t.Fatalf("calls=%d; want=%d", calls, wantCalls)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,4 +1,11 @@
|
||||
# Scert 整改任务执行清单
|
||||
### 19:13 执行补充:有界重试源码与回归完成
|
||||
|
||||
- [x] 只在公钥安装后私钥回连处接入重试,保留原私钥认证方式和已登记主机指纹,最多五次尝试。
|
||||
- [x] 恢复、耗尽、非可重试错误、取消路径回归及后端全量测试、静态检查、构建通过。
|
||||
- [~] 第二批竞态检查仍在运行;提交后构建正式包,继续密码 SSH 部署复测。
|
||||
- [ ] 远端更新服务器返回 200 与密钥状态 deployed 尚未确认,不新增生产放行或复审通过项。
|
||||
|
||||
### 19:10 执行补充
|
||||
|
||||
- [x] 部署、接口、应用包竞态检查全部通过,退出 0。
|
||||
|
||||
@ -1,4 +1,10 @@
|
||||
# Scert 整改、合并与优化清单
|
||||
### 19:13 开发补充:SSH 重载间隙处理已实现
|
||||
|
||||
- [x] 公钥安装后的私钥回连已增加最多五次尝试,间隔为 100、200、400、800 毫秒;只重试底层拨号连接拒绝,不重试认证失败、指纹不匹配或其他网络错误。
|
||||
- [x] 新增恢复成功、次数耗尽、认证失败、指纹失败、超时、非拨号错误、开始前取消及等待中取消回归;后端全量测试、静态检查和构建通过。
|
||||
- [~] 本批竞态检查正在运行,正式候选尚未构建部署。开发完成不等于该项实机复审通过;总体验收状态不变。
|
||||
|
||||
### 19:10 补充:回连失败现场已定位到 SSH 重载窗口
|
||||
|
||||
- [x] 三个包的竞态检查全部通过:部署 9.381 秒、接口 172.202 秒、应用 74.521 秒,命令退出 0。
|
||||
|
||||
Loading…
Reference in New Issue
Block a user