diff --git a/backend/internal/httpapi/server.go b/backend/internal/httpapi/server.go index 011a2bb..5b421e6 100644 --- a/backend/internal/httpapi/server.go +++ b/backend/internal/httpapi/server.go @@ -1000,7 +1000,7 @@ func (s *Server) provisionServerSSHKey(ctx context.Context, item deployment.Serv } privateItem := item privateItem.AuthMethod = deployment.SSHAuthPrivateKey - privateClient, err := s.connectSSHForKey(ctx, privateItem) + privateClient, err := s.connectSSHAfterReload(ctx, privateItem) if err != nil { s.logSSHProvisionFailure("private_key_connect", item, err) item.SSHKeyStatus = deployment.SSHKeyStatusFailed diff --git a/backend/internal/httpapi/ssh_reload.go b/backend/internal/httpapi/ssh_reload.go new file mode 100644 index 0000000..3a2dd31 --- /dev/null +++ b/backend/internal/httpapi/ssh_reload.go @@ -0,0 +1,34 @@ +package httpapi + +import ( + "context" + "errors" + "net" + "syscall" + "time" + + "git.pcux.cn/xuans/Scert/backend/internal/deployment" +) + +func (s *Server) connectSSHAfterReload(ctx context.Context, item deployment.Server) (deployment.Client, error) { + for attempt := 0; ; attempt++ { + if err := ctx.Err(); err != nil { + return nil, err + } + client, err := s.connectSSHForKey(ctx, item) + if err == nil { + return client, nil + } + var networkError *net.OpError + if attempt >= 4 || !errors.As(err, &networkError) || networkError.Op != "dial" || !errors.Is(err, syscall.ECONNREFUSED) { + return nil, err + } + timer := time.NewTimer((100 * time.Millisecond) << attempt) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } +} diff --git a/backend/internal/httpapi/ssh_reload_test.go b/backend/internal/httpapi/ssh_reload_test.go new file mode 100644 index 0000000..4ced975 --- /dev/null +++ b/backend/internal/httpapi/ssh_reload_test.go @@ -0,0 +1,98 @@ +package httpapi + +import ( + "context" + "errors" + "fmt" + "net" + "syscall" + "testing" + + "git.pcux.cn/xuans/Scert/backend/internal/deployment" +) + +type reloadTestFactory struct { + connect func(context.Context, deployment.Server) (deployment.Client, error) +} + +func (f reloadTestFactory) Connect(ctx context.Context, item deployment.Server) (deployment.Client, error) { + return f.connect(ctx, item) +} + +type reloadTestClient struct{ deployment.Client } + +func refusedReloadConnection() error { + return fmt.Errorf("%w", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}) +} + +func TestSSHAfterReloadRetriesOnlyConnectionRefusal(t *testing.T) { + for _, tc := range []struct { + name string + err error + failures int + calls int + }{ + {"ready", nil, 0, 1}, + {"recovered", refusedReloadConnection(), 2, 3}, + {"exhausted", refusedReloadConnection(), 5, 5}, + {"authentication", errors.New("authentication"), 1, 1}, + {"fingerprint", errors.New("fingerprint"), 1, 1}, + {"timeout", &net.OpError{Op: "dial", Err: syscall.ETIMEDOUT}, 1, 1}, + {"handshake", &net.OpError{Op: "read", Err: syscall.ECONNREFUSED}, 1, 1}, + } { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + wantClient := &reloadTestClient{} + server := &Server{deploymentFactory: reloadTestFactory{connect: func(ctx context.Context, item deployment.Server) (deployment.Client, error) { + calls++ + if item.AuthMethod != deployment.SSHAuthPrivateKey || item.HostKeyFingerprint != "SHA256:registered" { + t.Fatal(item.AuthMethod, item.HostKeyFingerprint) + } + if calls <= tc.failures { + return nil, tc.err + } + return wantClient, nil + }}} + client, err := server.connectSSHAfterReload(t.Context(), deployment.Server{ + AuthMethod: deployment.SSHAuthPrivateKey, HostKeyFingerprint: "SHA256:registered", + }) + if calls != tc.calls { + t.Fatalf("calls=%d; want=%d", calls, tc.calls) + } + if tc.calls <= tc.failures { + if !errors.Is(err, tc.err) || client != nil { + t.Fatal(client, err) + } + } else if err != nil || client != wantClient { + t.Fatal(client, err) + } + }) + } +} + +func TestSSHAfterReloadHonorsCancellation(t *testing.T) { + for _, cancelBeforeConnect := range []bool{false, true} { + ctx, cancel := context.WithCancel(t.Context()) + calls := 0 + server := &Server{deploymentFactory: reloadTestFactory{connect: func(context.Context, deployment.Server) (deployment.Client, error) { + calls++ + cancel() + return nil, refusedReloadConnection() + }}} + if cancelBeforeConnect { + cancel() + } + client, err := server.connectSSHAfterReload(ctx, deployment.Server{}) + cancel() + if !errors.Is(err, context.Canceled) || client != nil { + t.Fatal(client, err) + } + wantCalls := 1 + if cancelBeforeConnect { + wantCalls = 0 + } + if calls != wantCalls { + t.Fatalf("calls=%d; want=%d", calls, wantCalls) + } + } +} diff --git a/doc/Scert整改任务执行清单.md b/doc/Scert整改任务执行清单.md index 8589921..83a9dbd 100644 --- a/doc/Scert整改任务执行清单.md +++ b/doc/Scert整改任务执行清单.md @@ -1,4 +1,11 @@ # Scert 整改任务执行清单 +### 19:13 执行补充:有界重试源码与回归完成 + +- [x] 只在公钥安装后私钥回连处接入重试,保留原私钥认证方式和已登记主机指纹,最多五次尝试。 +- [x] 恢复、耗尽、非可重试错误、取消路径回归及后端全量测试、静态检查、构建通过。 +- [~] 第二批竞态检查仍在运行;提交后构建正式包,继续密码 SSH 部署复测。 +- [ ] 远端更新服务器返回 200 与密钥状态 deployed 尚未确认,不新增生产放行或复审通过项。 + ### 19:10 执行补充 - [x] 部署、接口、应用包竞态检查全部通过,退出 0。 diff --git a/doc/Scert整改合并优化清单.md b/doc/Scert整改合并优化清单.md index 778d7cc..94976ed 100644 --- a/doc/Scert整改合并优化清单.md +++ b/doc/Scert整改合并优化清单.md @@ -1,4 +1,10 @@ # Scert 整改、合并与优化清单 +### 19:13 开发补充:SSH 重载间隙处理已实现 + +- [x] 公钥安装后的私钥回连已增加最多五次尝试,间隔为 100、200、400、800 毫秒;只重试底层拨号连接拒绝,不重试认证失败、指纹不匹配或其他网络错误。 +- [x] 新增恢复成功、次数耗尽、认证失败、指纹失败、超时、非拨号错误、开始前取消及等待中取消回归;后端全量测试、静态检查和构建通过。 +- [~] 本批竞态检查正在运行,正式候选尚未构建部署。开发完成不等于该项实机复审通过;总体验收状态不变。 + ### 19:10 补充:回连失败现场已定位到 SSH 重载窗口 - [x] 三个包的竞态检查全部通过:部署 9.381 秒、接口 172.202 秒、应用 74.521 秒,命令退出 0。