From f95d82eca94954f7d8012d1b5537f8a0e02c3e59 Mon Sep 17 00:00:00 2001 From: georgehao Date: Wed, 18 Nov 2026 16:28:11 +0800 Subject: [PATCH 1/2] fix assign to same prover after failed --- common/version/version.go | 2 +- .../logic/provertask/batch_prover_task.go | 14 ------------ .../logic/provertask/bundle_prover_task.go | 22 +++---------------- .../logic/provertask/chunk_prover_task.go | 15 ------------- 4 files changed, 4 insertions(+), 49 deletions(-) diff --git a/common/version/version.go b/common/version/version.go index 3800664461..05833cc590 100644 --- a/common/version/version.go +++ b/common/version/version.go @@ -5,7 +5,7 @@ import ( "runtime/debug" ) -var tag = "v4.8.9" +var tag = "v4.9.0" var commit = func() string { if info, ok := debug.ReadBuildInfo(); ok { diff --git a/coordinator/internal/logic/provertask/batch_prover_task.go b/coordinator/internal/logic/provertask/batch_prover_task.go index 8f9061e14e..93865efcd3 100644 --- a/coordinator/internal/logic/provertask/batch_prover_task.go +++ b/coordinator/internal/logic/provertask/batch_prover_task.go @@ -144,20 +144,6 @@ func (bp *BatchProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinato // we are simply pick the chunk which has been assigned, so don't bother to update attempts or check failed before if taskCtx.hasAssignedTask == nil { - // Don't dispatch the same failing job to the same prover - proverTasks, getFailedTaskError := bp.proverTaskOrm.GetFailedProverTasksByHash(ctx.Copy(), message.ProofTypeBatch, tmpBatchTask.Hash, 2) - if getFailedTaskError != nil { - log.Error("failed to get prover tasks", "proof type", message.ProofTypeBatch.String(), "task ID", tmpBatchTask.Hash, "error", getFailedTaskError) - return nil, ErrCoordinatorInternalFailure - } - for i := 0; i < len(proverTasks); i++ { - if proverTasks[i].ProverPublicKey == taskCtx.PublicKey || - taskCtx.ProverProviderType == uint8(coordinatorType.ProverProviderTypeExternal) && cutils.IsExternalProverNameMatch(proverTasks[i].ProverName, taskCtx.ProverName) { - log.Debug("get empty batch, the prover already failed this task", "height", getTaskParameter.ProverHeight, "task ID", tmpBatchTask.Hash, "prover name", taskCtx.ProverName, "prover public key", taskCtx.PublicKey) - return nil, nil - } - } - rowsAffected, updateAttemptsErr := bp.batchOrm.UpdateBatchAttempts(ctx.Copy(), tmpBatchTask.Index, tmpBatchTask.ActiveAttempts, tmpBatchTask.TotalAttempts) if updateAttemptsErr != nil { log.Error("failed to update batch attempts", "height", getTaskParameter.ProverHeight, "err", updateAttemptsErr) diff --git a/coordinator/internal/logic/provertask/bundle_prover_task.go b/coordinator/internal/logic/provertask/bundle_prover_task.go index f40ae3dcdc..b529403717 100644 --- a/coordinator/internal/logic/provertask/bundle_prover_task.go +++ b/coordinator/internal/logic/provertask/bundle_prover_task.go @@ -14,14 +14,12 @@ import ( "github.com/scroll-tech/go-ethereum/params" "gorm.io/gorm" - "scroll-tech/coordinator/internal/config" - "scroll-tech/coordinator/internal/orm" - coordinatorType "scroll-tech/coordinator/internal/types" - cutils "scroll-tech/coordinator/internal/utils" - "scroll-tech/common/types" "scroll-tech/common/types/message" "scroll-tech/common/utils" + "scroll-tech/coordinator/internal/config" + "scroll-tech/coordinator/internal/orm" + coordinatorType "scroll-tech/coordinator/internal/types" ) // BundleProverTask is prover task implement for bundle proof @@ -142,20 +140,6 @@ func (bp *BundleProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinat // we are simply pick the chunk which has been assigned, so don't bother to update attempts or check failed before if taskCtx.hasAssignedTask == nil { - // Don't dispatch the same failing job to the same prover - proverTasks, getTaskError := bp.proverTaskOrm.GetFailedProverTasksByHash(ctx.Copy(), message.ProofTypeBundle, tmpBundleTask.Hash, 2) - if getTaskError != nil { - log.Error("failed to get prover tasks", "proof type", message.ProofTypeBundle.String(), "task ID", tmpBundleTask.Hash, "error", getTaskError) - return nil, ErrCoordinatorInternalFailure - } - for i := 0; i < len(proverTasks); i++ { - if proverTasks[i].ProverPublicKey == taskCtx.PublicKey || - taskCtx.ProverProviderType == uint8(coordinatorType.ProverProviderTypeExternal) && cutils.IsExternalProverNameMatch(proverTasks[i].ProverName, taskCtx.ProverName) { - log.Debug("get empty bundle, the prover already failed this task", "height", getTaskParameter.ProverHeight, "task ID", tmpBundleTask.Hash, "prover name", taskCtx.ProverName, "prover public key", taskCtx.PublicKey) - return nil, nil - } - } - rowsAffected, updateAttemptsErr := bp.bundleOrm.UpdateBundleAttempts(ctx.Copy(), tmpBundleTask.Hash, tmpBundleTask.ActiveAttempts, tmpBundleTask.TotalAttempts) if updateAttemptsErr != nil { log.Error("failed to update bundle attempts", "height", getTaskParameter.ProverHeight, "err", updateAttemptsErr) diff --git a/coordinator/internal/logic/provertask/chunk_prover_task.go b/coordinator/internal/logic/provertask/chunk_prover_task.go index 6492c10a10..056a86a20a 100644 --- a/coordinator/internal/logic/provertask/chunk_prover_task.go +++ b/coordinator/internal/logic/provertask/chunk_prover_task.go @@ -21,7 +21,6 @@ import ( "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/orm" coordinatorType "scroll-tech/coordinator/internal/types" - cutils "scroll-tech/coordinator/internal/utils" ) // ChunkProverTask the chunk prover task @@ -140,20 +139,6 @@ func (cp *ChunkProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinato // we are simply pick the chunk which has been assigned, so don't bother to update attempts or check failed before if taskCtx.hasAssignedTask == nil { - // Don't dispatch the same failing job to the same prover - proverTasks, getFailedTaskError := cp.proverTaskOrm.GetFailedProverTasksByHash(ctx.Copy(), message.ProofTypeChunk, tmpChunkTask.Hash, 2) - if getFailedTaskError != nil { - log.Error("failed to get prover tasks", "proof type", message.ProofTypeChunk.String(), "task ID", tmpChunkTask.Hash, "error", getFailedTaskError) - return nil, ErrCoordinatorInternalFailure - } - for i := 0; i < len(proverTasks); i++ { - if proverTasks[i].ProverPublicKey == taskCtx.PublicKey || - taskCtx.ProverProviderType == uint8(coordinatorType.ProverProviderTypeExternal) && cutils.IsExternalProverNameMatch(proverTasks[i].ProverName, taskCtx.ProverName) { - log.Debug("get empty chunk, the prover already failed this task", "height", getTaskParameter.ProverHeight, "task ID", tmpChunkTask.Hash, "prover name", taskCtx.ProverName, "prover public key", taskCtx.PublicKey) - return nil, nil - } - } - rowsAffected, updateAttemptsErr := cp.chunkOrm.UpdateChunkAttempts(ctx.Copy(), tmpChunkTask.Index, tmpChunkTask.ActiveAttempts, tmpChunkTask.TotalAttempts) if updateAttemptsErr != nil { log.Error("failed to update chunk attempts", "height", getTaskParameter.ProverHeight, "err", updateAttemptsErr) From 5fd64e37abfcf0f1c6d75aacb18ab12c009ebef9 Mon Sep 17 00:00:00 2001 From: georgehao Date: Wed, 18 Nov 2026 16:38:47 +0800 Subject: [PATCH 2/2] update --- coordinator/conf/config.json | 1 + coordinator/internal/config/config.go | 2 ++ .../logic/provertask/batch_prover_task.go | 16 +++++++++++++ .../logic/provertask/bundle_prover_task.go | 24 ++++++++++++++++--- .../logic/provertask/chunk_prover_task.go | 17 +++++++++++++ 5 files changed, 57 insertions(+), 3 deletions(-) diff --git a/coordinator/conf/config.json b/coordinator/conf/config.json index 7fa8e75503..c2431f666c 100644 --- a/coordinator/conf/config.json +++ b/coordinator/conf/config.json @@ -2,6 +2,7 @@ "prover_manager": { "provers_per_session": 1, "session_attempts": 5, + "skip_failed_provers": false, "external_prover_threshold": 32, "bundle_collection_time_sec": 180, "batch_collection_time_sec": 180, diff --git a/coordinator/internal/config/config.go b/coordinator/internal/config/config.go index 3364931b46..71ed70925a 100644 --- a/coordinator/internal/config/config.go +++ b/coordinator/internal/config/config.go @@ -16,6 +16,8 @@ type ProverManager struct { // Number of attempts that a session can be retried if previous attempts failed. // Currently we only consider proving timeout as failure here. SessionAttempts uint8 `json:"session_attempts"` + // SkipFailedProvers enables filtering based on recent failures of the same task. + SkipFailedProvers bool `json:"skip_failed_provers"` // Threshold for activating the external prover based on unassigned task count. ExternalProverThreshold int64 `json:"external_prover_threshold"` // Zk verifier config. diff --git a/coordinator/internal/logic/provertask/batch_prover_task.go b/coordinator/internal/logic/provertask/batch_prover_task.go index 93865efcd3..561307d994 100644 --- a/coordinator/internal/logic/provertask/batch_prover_task.go +++ b/coordinator/internal/logic/provertask/batch_prover_task.go @@ -144,6 +144,22 @@ func (bp *BatchProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinato // we are simply pick the chunk which has been assigned, so don't bother to update attempts or check failed before if taskCtx.hasAssignedTask == nil { + if bp.cfg.ProverManager.SkipFailedProvers { + // Don't dispatch the same failing job to the same prover + proverTasks, getFailedTaskError := bp.proverTaskOrm.GetFailedProverTasksByHash(ctx.Copy(), message.ProofTypeBatch, tmpBatchTask.Hash, 2) + if getFailedTaskError != nil { + log.Error("failed to get prover tasks", "proof type", message.ProofTypeBatch.String(), "task ID", tmpBatchTask.Hash, "error", getFailedTaskError) + return nil, ErrCoordinatorInternalFailure + } + for i := 0; i < len(proverTasks); i++ { + if proverTasks[i].ProverPublicKey == taskCtx.PublicKey || + taskCtx.ProverProviderType == uint8(coordinatorType.ProverProviderTypeExternal) && cutils.IsExternalProverNameMatch(proverTasks[i].ProverName, taskCtx.ProverName) { + log.Debug("get empty batch, the prover already failed this task", "height", getTaskParameter.ProverHeight, "task ID", tmpBatchTask.Hash, "prover name", taskCtx.ProverName, "prover public key", taskCtx.PublicKey) + return nil, nil + } + } + } + rowsAffected, updateAttemptsErr := bp.batchOrm.UpdateBatchAttempts(ctx.Copy(), tmpBatchTask.Index, tmpBatchTask.ActiveAttempts, tmpBatchTask.TotalAttempts) if updateAttemptsErr != nil { log.Error("failed to update batch attempts", "height", getTaskParameter.ProverHeight, "err", updateAttemptsErr) diff --git a/coordinator/internal/logic/provertask/bundle_prover_task.go b/coordinator/internal/logic/provertask/bundle_prover_task.go index b529403717..20e61a5a08 100644 --- a/coordinator/internal/logic/provertask/bundle_prover_task.go +++ b/coordinator/internal/logic/provertask/bundle_prover_task.go @@ -14,12 +14,14 @@ import ( "github.com/scroll-tech/go-ethereum/params" "gorm.io/gorm" - "scroll-tech/common/types" - "scroll-tech/common/types/message" - "scroll-tech/common/utils" "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/orm" coordinatorType "scroll-tech/coordinator/internal/types" + cutils "scroll-tech/coordinator/internal/utils" + + "scroll-tech/common/types" + "scroll-tech/common/types/message" + "scroll-tech/common/utils" ) // BundleProverTask is prover task implement for bundle proof @@ -140,6 +142,22 @@ func (bp *BundleProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinat // we are simply pick the chunk which has been assigned, so don't bother to update attempts or check failed before if taskCtx.hasAssignedTask == nil { + if bp.cfg.ProverManager.SkipFailedProvers { + // Don't dispatch the same failing job to the same prover + proverTasks, getFailedTaskError := bp.proverTaskOrm.GetFailedProverTasksByHash(ctx.Copy(), message.ProofTypeBundle, tmpBundleTask.Hash, 2) + if getFailedTaskError != nil { + log.Error("failed to get prover tasks", "proof type", message.ProofTypeBundle.String(), "task ID", tmpBundleTask.Hash, "error", getFailedTaskError) + return nil, ErrCoordinatorInternalFailure + } + for i := 0; i < len(proverTasks); i++ { + if proverTasks[i].ProverPublicKey == taskCtx.PublicKey || + taskCtx.ProverProviderType == uint8(coordinatorType.ProverProviderTypeExternal) && cutils.IsExternalProverNameMatch(proverTasks[i].ProverName, taskCtx.ProverName) { + log.Debug("get empty bundle, the prover already failed this task", "height", getTaskParameter.ProverHeight, "task ID", tmpBundleTask.Hash, "prover name", taskCtx.ProverName, "prover public key", taskCtx.PublicKey) + return nil, nil + } + } + } + rowsAffected, updateAttemptsErr := bp.bundleOrm.UpdateBundleAttempts(ctx.Copy(), tmpBundleTask.Hash, tmpBundleTask.ActiveAttempts, tmpBundleTask.TotalAttempts) if updateAttemptsErr != nil { log.Error("failed to update bundle attempts", "height", getTaskParameter.ProverHeight, "err", updateAttemptsErr) diff --git a/coordinator/internal/logic/provertask/chunk_prover_task.go b/coordinator/internal/logic/provertask/chunk_prover_task.go index 056a86a20a..ae0b24292e 100644 --- a/coordinator/internal/logic/provertask/chunk_prover_task.go +++ b/coordinator/internal/logic/provertask/chunk_prover_task.go @@ -21,6 +21,7 @@ import ( "scroll-tech/coordinator/internal/config" "scroll-tech/coordinator/internal/orm" coordinatorType "scroll-tech/coordinator/internal/types" + cutils "scroll-tech/coordinator/internal/utils" ) // ChunkProverTask the chunk prover task @@ -139,6 +140,22 @@ func (cp *ChunkProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinato // we are simply pick the chunk which has been assigned, so don't bother to update attempts or check failed before if taskCtx.hasAssignedTask == nil { + if cp.cfg.ProverManager.SkipFailedProvers { + // Don't dispatch the same failing job to the same prover + proverTasks, getFailedTaskError := cp.proverTaskOrm.GetFailedProverTasksByHash(ctx.Copy(), message.ProofTypeChunk, tmpChunkTask.Hash, 2) + if getFailedTaskError != nil { + log.Error("failed to get prover tasks", "proof type", message.ProofTypeChunk.String(), "task ID", tmpChunkTask.Hash, "error", getFailedTaskError) + return nil, ErrCoordinatorInternalFailure + } + for i := 0; i < len(proverTasks); i++ { + if proverTasks[i].ProverPublicKey == taskCtx.PublicKey || + taskCtx.ProverProviderType == uint8(coordinatorType.ProverProviderTypeExternal) && cutils.IsExternalProverNameMatch(proverTasks[i].ProverName, taskCtx.ProverName) { + log.Debug("get empty chunk, the prover already failed this task", "height", getTaskParameter.ProverHeight, "task ID", tmpChunkTask.Hash, "prover name", taskCtx.ProverName, "prover public key", taskCtx.PublicKey) + return nil, nil + } + } + } + rowsAffected, updateAttemptsErr := cp.chunkOrm.UpdateChunkAttempts(ctx.Copy(), tmpChunkTask.Index, tmpChunkTask.ActiveAttempts, tmpChunkTask.TotalAttempts) if updateAttemptsErr != nil { log.Error("failed to update chunk attempts", "height", getTaskParameter.ProverHeight, "err", updateAttemptsErr)