修复SSH重载后私钥回连的短暂连接拒绝
This commit is contained in:
parent
1a317c2175
commit
714753236f
@ -1000,7 +1000,7 @@ func (s *Server) provisionServerSSHKey(ctx context.Context, item deployment.Serv
|
|||||||
}
|
}
|
||||||
privateItem := item
|
privateItem := item
|
||||||
privateItem.AuthMethod = deployment.SSHAuthPrivateKey
|
privateItem.AuthMethod = deployment.SSHAuthPrivateKey
|
||||||
privateClient, err := s.connectSSHForKey(ctx, privateItem)
|
privateClient, err := s.connectSSHAfterReload(ctx, privateItem)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logSSHProvisionFailure("private_key_connect", item, err)
|
s.logSSHProvisionFailure("private_key_connect", item, err)
|
||||||
item.SSHKeyStatus = deployment.SSHKeyStatusFailed
|
item.SSHKeyStatus = deployment.SSHKeyStatusFailed
|
||||||
|
|||||||
34
backend/internal/httpapi/ssh_reload.go
Normal file
34
backend/internal/httpapi/ssh_reload.go
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (s *Server) connectSSHAfterReload(ctx context.Context, item deployment.Server) (deployment.Client, error) {
|
||||||
|
for attempt := 0; ; attempt++ {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
client, err := s.connectSSHForKey(ctx, item)
|
||||||
|
if err == nil {
|
||||||
|
return client, nil
|
||||||
|
}
|
||||||
|
var networkError *net.OpError
|
||||||
|
if attempt >= 4 || !errors.As(err, &networkError) || networkError.Op != "dial" || !errors.Is(err, syscall.ECONNREFUSED) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
timer := time.NewTimer((100 * time.Millisecond) << attempt)
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
timer.Stop()
|
||||||
|
return nil, ctx.Err()
|
||||||
|
case <-timer.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
98
backend/internal/httpapi/ssh_reload_test.go
Normal file
98
backend/internal/httpapi/ssh_reload_test.go
Normal file
@ -0,0 +1,98 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.pcux.cn/xuans/Scert/backend/internal/deployment"
|
||||||
|
)
|
||||||
|
|
||||||
|
type reloadTestFactory struct {
|
||||||
|
connect func(context.Context, deployment.Server) (deployment.Client, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f reloadTestFactory) Connect(ctx context.Context, item deployment.Server) (deployment.Client, error) {
|
||||||
|
return f.connect(ctx, item)
|
||||||
|
}
|
||||||
|
|
||||||
|
type reloadTestClient struct{ deployment.Client }
|
||||||
|
|
||||||
|
func refusedReloadConnection() error {
|
||||||
|
return fmt.Errorf("%w", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSSHAfterReloadRetriesOnlyConnectionRefusal(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
err error
|
||||||
|
failures int
|
||||||
|
calls int
|
||||||
|
}{
|
||||||
|
{"ready", nil, 0, 1},
|
||||||
|
{"recovered", refusedReloadConnection(), 2, 3},
|
||||||
|
{"exhausted", refusedReloadConnection(), 5, 5},
|
||||||
|
{"authentication", errors.New("authentication"), 1, 1},
|
||||||
|
{"fingerprint", errors.New("fingerprint"), 1, 1},
|
||||||
|
{"timeout", &net.OpError{Op: "dial", Err: syscall.ETIMEDOUT}, 1, 1},
|
||||||
|
{"handshake", &net.OpError{Op: "read", Err: syscall.ECONNREFUSED}, 1, 1},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
calls := 0
|
||||||
|
wantClient := &reloadTestClient{}
|
||||||
|
server := &Server{deploymentFactory: reloadTestFactory{connect: func(ctx context.Context, item deployment.Server) (deployment.Client, error) {
|
||||||
|
calls++
|
||||||
|
if item.AuthMethod != deployment.SSHAuthPrivateKey || item.HostKeyFingerprint != "SHA256:registered" {
|
||||||
|
t.Fatal(item.AuthMethod, item.HostKeyFingerprint)
|
||||||
|
}
|
||||||
|
if calls <= tc.failures {
|
||||||
|
return nil, tc.err
|
||||||
|
}
|
||||||
|
return wantClient, nil
|
||||||
|
}}}
|
||||||
|
client, err := server.connectSSHAfterReload(t.Context(), deployment.Server{
|
||||||
|
AuthMethod: deployment.SSHAuthPrivateKey, HostKeyFingerprint: "SHA256:registered",
|
||||||
|
})
|
||||||
|
if calls != tc.calls {
|
||||||
|
t.Fatalf("calls=%d; want=%d", calls, tc.calls)
|
||||||
|
}
|
||||||
|
if tc.calls <= tc.failures {
|
||||||
|
if !errors.Is(err, tc.err) || client != nil {
|
||||||
|
t.Fatal(client, err)
|
||||||
|
}
|
||||||
|
} else if err != nil || client != wantClient {
|
||||||
|
t.Fatal(client, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSSHAfterReloadHonorsCancellation(t *testing.T) {
|
||||||
|
for _, cancelBeforeConnect := range []bool{false, true} {
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
calls := 0
|
||||||
|
server := &Server{deploymentFactory: reloadTestFactory{connect: func(context.Context, deployment.Server) (deployment.Client, error) {
|
||||||
|
calls++
|
||||||
|
cancel()
|
||||||
|
return nil, refusedReloadConnection()
|
||||||
|
}}}
|
||||||
|
if cancelBeforeConnect {
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
client, err := server.connectSSHAfterReload(ctx, deployment.Server{})
|
||||||
|
cancel()
|
||||||
|
if !errors.Is(err, context.Canceled) || client != nil {
|
||||||
|
t.Fatal(client, err)
|
||||||
|
}
|
||||||
|
wantCalls := 1
|
||||||
|
if cancelBeforeConnect {
|
||||||
|
wantCalls = 0
|
||||||
|
}
|
||||||
|
if calls != wantCalls {
|
||||||
|
t.Fatalf("calls=%d; want=%d", calls, wantCalls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,4 +1,11 @@
|
|||||||
# Scert 整改任务执行清单
|
# Scert 整改任务执行清单
|
||||||
|
### 19:13 执行补充:有界重试源码与回归完成
|
||||||
|
|
||||||
|
- [x] 只在公钥安装后私钥回连处接入重试,保留原私钥认证方式和已登记主机指纹,最多五次尝试。
|
||||||
|
- [x] 恢复、耗尽、非可重试错误、取消路径回归及后端全量测试、静态检查、构建通过。
|
||||||
|
- [~] 第二批竞态检查仍在运行;提交后构建正式包,继续密码 SSH 部署复测。
|
||||||
|
- [ ] 远端更新服务器返回 200 与密钥状态 deployed 尚未确认,不新增生产放行或复审通过项。
|
||||||
|
|
||||||
### 19:10 执行补充
|
### 19:10 执行补充
|
||||||
|
|
||||||
- [x] 部署、接口、应用包竞态检查全部通过,退出 0。
|
- [x] 部署、接口、应用包竞态检查全部通过,退出 0。
|
||||||
|
|||||||
@ -1,4 +1,10 @@
|
|||||||
# Scert 整改、合并与优化清单
|
# Scert 整改、合并与优化清单
|
||||||
|
### 19:13 开发补充:SSH 重载间隙处理已实现
|
||||||
|
|
||||||
|
- [x] 公钥安装后的私钥回连已增加最多五次尝试,间隔为 100、200、400、800 毫秒;只重试底层拨号连接拒绝,不重试认证失败、指纹不匹配或其他网络错误。
|
||||||
|
- [x] 新增恢复成功、次数耗尽、认证失败、指纹失败、超时、非拨号错误、开始前取消及等待中取消回归;后端全量测试、静态检查和构建通过。
|
||||||
|
- [~] 本批竞态检查正在运行,正式候选尚未构建部署。开发完成不等于该项实机复审通过;总体验收状态不变。
|
||||||
|
|
||||||
### 19:10 补充:回连失败现场已定位到 SSH 重载窗口
|
### 19:10 补充:回连失败现场已定位到 SSH 重载窗口
|
||||||
|
|
||||||
- [x] 三个包的竞态检查全部通过:部署 9.381 秒、接口 172.202 秒、应用 74.521 秒,命令退出 0。
|
- [x] 三个包的竞态检查全部通过:部署 9.381 秒、接口 172.202 秒、应用 74.521 秒,命令退出 0。
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user