修复SSH重载后私钥回连的短暂连接拒绝

This commit is contained in:
xuans 2026-09-30 19:12:44 +08:00
parent 1a317c2175
commit 714753236f
5 changed files with 146 additions and 1 deletions

View File

@ -1000,7 +1000,7 @@ func (s *Server) provisionServerSSHKey(ctx context.Context, item deployment.Serv
}
privateItem := item
privateItem.AuthMethod = deployment.SSHAuthPrivateKey
privateClient, err := s.connectSSHForKey(ctx, privateItem)
privateClient, err := s.connectSSHAfterReload(ctx, privateItem)
if err != nil {
s.logSSHProvisionFailure("private_key_connect", item, err)
item.SSHKeyStatus = deployment.SSHKeyStatusFailed

View File

@ -0,0 +1,34 @@
package httpapi
import (
"context"
"errors"
"net"
"syscall"
"time"
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
)
func (s *Server) connectSSHAfterReload(ctx context.Context, item deployment.Server) (deployment.Client, error) {
for attempt := 0; ; attempt++ {
if err := ctx.Err(); err != nil {
return nil, err
}
client, err := s.connectSSHForKey(ctx, item)
if err == nil {
return client, nil
}
var networkError *net.OpError
if attempt >= 4 || !errors.As(err, &networkError) || networkError.Op != "dial" || !errors.Is(err, syscall.ECONNREFUSED) {
return nil, err
}
timer := time.NewTimer((100 * time.Millisecond) << attempt)
select {
case <-ctx.Done():
timer.Stop()
return nil, ctx.Err()
case <-timer.C:
}
}
}

View File

@ -0,0 +1,98 @@
package httpapi
import (
"context"
"errors"
"fmt"
"net"
"syscall"
"testing"
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
)
type reloadTestFactory struct {
connect func(context.Context, deployment.Server) (deployment.Client, error)
}
func (f reloadTestFactory) Connect(ctx context.Context, item deployment.Server) (deployment.Client, error) {
return f.connect(ctx, item)
}
type reloadTestClient struct{ deployment.Client }
func refusedReloadConnection() error {
return fmt.Errorf("%w", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED})
}
func TestSSHAfterReloadRetriesOnlyConnectionRefusal(t *testing.T) {
for _, tc := range []struct {
name string
err error
failures int
calls int
}{
{"ready", nil, 0, 1},
{"recovered", refusedReloadConnection(), 2, 3},
{"exhausted", refusedReloadConnection(), 5, 5},
{"authentication", errors.New("authentication"), 1, 1},
{"fingerprint", errors.New("fingerprint"), 1, 1},
{"timeout", &net.OpError{Op: "dial", Err: syscall.ETIMEDOUT}, 1, 1},
{"handshake", &net.OpError{Op: "read", Err: syscall.ECONNREFUSED}, 1, 1},
} {
t.Run(tc.name, func(t *testing.T) {
calls := 0
wantClient := &reloadTestClient{}
server := &Server{deploymentFactory: reloadTestFactory{connect: func(ctx context.Context, item deployment.Server) (deployment.Client, error) {
calls++
if item.AuthMethod != deployment.SSHAuthPrivateKey || item.HostKeyFingerprint != "SHA256:registered" {
t.Fatal(item.AuthMethod, item.HostKeyFingerprint)
}
if calls <= tc.failures {
return nil, tc.err
}
return wantClient, nil
}}}
client, err := server.connectSSHAfterReload(t.Context(), deployment.Server{
AuthMethod: deployment.SSHAuthPrivateKey, HostKeyFingerprint: "SHA256:registered",
})
if calls != tc.calls {
t.Fatalf("calls=%d; want=%d", calls, tc.calls)
}
if tc.calls <= tc.failures {
if !errors.Is(err, tc.err) || client != nil {
t.Fatal(client, err)
}
} else if err != nil || client != wantClient {
t.Fatal(client, err)
}
})
}
}
func TestSSHAfterReloadHonorsCancellation(t *testing.T) {
for _, cancelBeforeConnect := range []bool{false, true} {
ctx, cancel := context.WithCancel(t.Context())
calls := 0
server := &Server{deploymentFactory: reloadTestFactory{connect: func(context.Context, deployment.Server) (deployment.Client, error) {
calls++
cancel()
return nil, refusedReloadConnection()
}}}
if cancelBeforeConnect {
cancel()
}
client, err := server.connectSSHAfterReload(ctx, deployment.Server{})
cancel()
if !errors.Is(err, context.Canceled) || client != nil {
t.Fatal(client, err)
}
wantCalls := 1
if cancelBeforeConnect {
wantCalls = 0
}
if calls != wantCalls {
t.Fatalf("calls=%d; want=%d", calls, wantCalls)
}
}
}

View File

@ -1,4 +1,11 @@
# Scert 整改任务执行清单
### 19:13 执行补充:有界重试源码与回归完成
- [x] 只在公钥安装后私钥回连处接入重试,保留原私钥认证方式和已登记主机指纹,最多五次尝试。
- [x] 恢复、耗尽、非可重试错误、取消路径回归及后端全量测试、静态检查、构建通过。
- [~] 第二批竞态检查仍在运行;提交后构建正式包,继续密码 SSH 部署复测。
- [ ] 远端更新服务器返回 200 与密钥状态 deployed 尚未确认,不新增生产放行或复审通过项。
### 19:10 执行补充
- [x] 部署、接口、应用包竞态检查全部通过,退出 0。

View File

@ -1,4 +1,10 @@
# Scert 整改、合并与优化清单
### 19:13 开发补充:SSH 重载间隙处理已实现
- [x] 公钥安装后的私钥回连已增加最多五次尝试,间隔为 100、200、400、800 毫秒;只重试底层拨号连接拒绝,不重试认证失败、指纹不匹配或其他网络错误。
- [x] 新增恢复成功、次数耗尽、认证失败、指纹失败、超时、非拨号错误、开始前取消及等待中取消回归;后端全量测试、静态检查和构建通过。
- [~] 本批竞态检查正在运行,正式候选尚未构建部署。开发完成不等于该项实机复审通过;总体验收状态不变。
### 19:10 补充:回连失败现场已定位到 SSH 重载窗口
- [x] 三个包的竞态检查全部通过:部署 9.381 秒、接口 172.202 秒、应用 74.521 秒,命令退出 0。