diff --git a/common/version/version.go b/common/version/version.go index 3800664461..05833cc590 100644 --- a/common/version/version.go +++ b/common/version/version.go @@ -5,7 +5,7 @@ import ( "runtime/debug" ) -var tag = "v4.8.9" +var tag = "v4.9.0" var commit = func() string { if info, ok := debug.ReadBuildInfo(); ok { diff --git a/coordinator/conf/config.json b/coordinator/conf/config.json index 7fa8e75503..c2431f666c 100644 --- a/coordinator/conf/config.json +++ b/coordinator/conf/config.json @@ -2,6 +2,7 @@ "prover_manager": { "provers_per_session": 1, "session_attempts": 5, + "skip_failed_provers": false, "external_prover_threshold": 32, "bundle_collection_time_sec": 180, "batch_collection_time_sec": 180, diff --git a/coordinator/internal/config/config.go b/coordinator/internal/config/config.go index 3364931b46..71ed70925a 100644 --- a/coordinator/internal/config/config.go +++ b/coordinator/internal/config/config.go @@ -16,6 +16,8 @@ type ProverManager struct { // Number of attempts that a session can be retried if previous attempts failed. // Currently we only consider proving timeout as failure here. SessionAttempts uint8 `json:"session_attempts"` + // SkipFailedProvers enables filtering based on recent failures of the same task. + SkipFailedProvers bool `json:"skip_failed_provers"` // Threshold for activating the external prover based on unassigned task count. ExternalProverThreshold int64 `json:"external_prover_threshold"` // Zk verifier config. diff --git a/coordinator/internal/logic/provertask/batch_prover_task.go b/coordinator/internal/logic/provertask/batch_prover_task.go index 8f9061e14e..561307d994 100644 --- a/coordinator/internal/logic/provertask/batch_prover_task.go +++ b/coordinator/internal/logic/provertask/batch_prover_task.go @@ -144,17 +144,19 @@ func (bp *BatchProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinato // we are simply pick the chunk which has been assigned, so don't bother to update attempts or check failed before if taskCtx.hasAssignedTask == nil { - // Don't dispatch the same failing job to the same prover - proverTasks, getFailedTaskError := bp.proverTaskOrm.GetFailedProverTasksByHash(ctx.Copy(), message.ProofTypeBatch, tmpBatchTask.Hash, 2) - if getFailedTaskError != nil { - log.Error("failed to get prover tasks", "proof type", message.ProofTypeBatch.String(), "task ID", tmpBatchTask.Hash, "error", getFailedTaskError) - return nil, ErrCoordinatorInternalFailure - } - for i := 0; i < len(proverTasks); i++ { - if proverTasks[i].ProverPublicKey == taskCtx.PublicKey || - taskCtx.ProverProviderType == uint8(coordinatorType.ProverProviderTypeExternal) && cutils.IsExternalProverNameMatch(proverTasks[i].ProverName, taskCtx.ProverName) { - log.Debug("get empty batch, the prover already failed this task", "height", getTaskParameter.ProverHeight, "task ID", tmpBatchTask.Hash, "prover name", taskCtx.ProverName, "prover public key", taskCtx.PublicKey) - return nil, nil + if bp.cfg.ProverManager.SkipFailedProvers { + // Don't dispatch the same failing job to the same prover + proverTasks, getFailedTaskError := bp.proverTaskOrm.GetFailedProverTasksByHash(ctx.Copy(), message.ProofTypeBatch, tmpBatchTask.Hash, 2) + if getFailedTaskError != nil { + log.Error("failed to get prover tasks", "proof type", message.ProofTypeBatch.String(), "task ID", tmpBatchTask.Hash, "error", getFailedTaskError) + return nil, ErrCoordinatorInternalFailure + } + for i := 0; i < len(proverTasks); i++ { + if proverTasks[i].ProverPublicKey == taskCtx.PublicKey || + taskCtx.ProverProviderType == uint8(coordinatorType.ProverProviderTypeExternal) && cutils.IsExternalProverNameMatch(proverTasks[i].ProverName, taskCtx.ProverName) { + log.Debug("get empty batch, the prover already failed this task", "height", getTaskParameter.ProverHeight, "task ID", tmpBatchTask.Hash, "prover name", taskCtx.ProverName, "prover public key", taskCtx.PublicKey) + return nil, nil + } } } diff --git a/coordinator/internal/logic/provertask/bundle_prover_task.go b/coordinator/internal/logic/provertask/bundle_prover_task.go index f40ae3dcdc..20e61a5a08 100644 --- a/coordinator/internal/logic/provertask/bundle_prover_task.go +++ b/coordinator/internal/logic/provertask/bundle_prover_task.go @@ -142,17 +142,19 @@ func (bp *BundleProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinat // we are simply pick the chunk which has been assigned, so don't bother to update attempts or check failed before if taskCtx.hasAssignedTask == nil { - // Don't dispatch the same failing job to the same prover - proverTasks, getTaskError := bp.proverTaskOrm.GetFailedProverTasksByHash(ctx.Copy(), message.ProofTypeBundle, tmpBundleTask.Hash, 2) - if getTaskError != nil { - log.Error("failed to get prover tasks", "proof type", message.ProofTypeBundle.String(), "task ID", tmpBundleTask.Hash, "error", getTaskError) - return nil, ErrCoordinatorInternalFailure - } - for i := 0; i < len(proverTasks); i++ { - if proverTasks[i].ProverPublicKey == taskCtx.PublicKey || - taskCtx.ProverProviderType == uint8(coordinatorType.ProverProviderTypeExternal) && cutils.IsExternalProverNameMatch(proverTasks[i].ProverName, taskCtx.ProverName) { - log.Debug("get empty bundle, the prover already failed this task", "height", getTaskParameter.ProverHeight, "task ID", tmpBundleTask.Hash, "prover name", taskCtx.ProverName, "prover public key", taskCtx.PublicKey) - return nil, nil + if bp.cfg.ProverManager.SkipFailedProvers { + // Don't dispatch the same failing job to the same prover + proverTasks, getFailedTaskError := bp.proverTaskOrm.GetFailedProverTasksByHash(ctx.Copy(), message.ProofTypeBundle, tmpBundleTask.Hash, 2) + if getFailedTaskError != nil { + log.Error("failed to get prover tasks", "proof type", message.ProofTypeBundle.String(), "task ID", tmpBundleTask.Hash, "error", getFailedTaskError) + return nil, ErrCoordinatorInternalFailure + } + for i := 0; i < len(proverTasks); i++ { + if proverTasks[i].ProverPublicKey == taskCtx.PublicKey || + taskCtx.ProverProviderType == uint8(coordinatorType.ProverProviderTypeExternal) && cutils.IsExternalProverNameMatch(proverTasks[i].ProverName, taskCtx.ProverName) { + log.Debug("get empty bundle, the prover already failed this task", "height", getTaskParameter.ProverHeight, "task ID", tmpBundleTask.Hash, "prover name", taskCtx.ProverName, "prover public key", taskCtx.PublicKey) + return nil, nil + } } } diff --git a/coordinator/internal/logic/provertask/chunk_prover_task.go b/coordinator/internal/logic/provertask/chunk_prover_task.go index 6492c10a10..ae0b24292e 100644 --- a/coordinator/internal/logic/provertask/chunk_prover_task.go +++ b/coordinator/internal/logic/provertask/chunk_prover_task.go @@ -140,17 +140,19 @@ func (cp *ChunkProverTask) Assign(ctx *gin.Context, getTaskParameter *coordinato // we are simply pick the chunk which has been assigned, so don't bother to update attempts or check failed before if taskCtx.hasAssignedTask == nil { - // Don't dispatch the same failing job to the same prover - proverTasks, getFailedTaskError := cp.proverTaskOrm.GetFailedProverTasksByHash(ctx.Copy(), message.ProofTypeChunk, tmpChunkTask.Hash, 2) - if getFailedTaskError != nil { - log.Error("failed to get prover tasks", "proof type", message.ProofTypeChunk.String(), "task ID", tmpChunkTask.Hash, "error", getFailedTaskError) - return nil, ErrCoordinatorInternalFailure - } - for i := 0; i < len(proverTasks); i++ { - if proverTasks[i].ProverPublicKey == taskCtx.PublicKey || - taskCtx.ProverProviderType == uint8(coordinatorType.ProverProviderTypeExternal) && cutils.IsExternalProverNameMatch(proverTasks[i].ProverName, taskCtx.ProverName) { - log.Debug("get empty chunk, the prover already failed this task", "height", getTaskParameter.ProverHeight, "task ID", tmpChunkTask.Hash, "prover name", taskCtx.ProverName, "prover public key", taskCtx.PublicKey) - return nil, nil + if cp.cfg.ProverManager.SkipFailedProvers { + // Don't dispatch the same failing job to the same prover + proverTasks, getFailedTaskError := cp.proverTaskOrm.GetFailedProverTasksByHash(ctx.Copy(), message.ProofTypeChunk, tmpChunkTask.Hash, 2) + if getFailedTaskError != nil { + log.Error("failed to get prover tasks", "proof type", message.ProofTypeChunk.String(), "task ID", tmpChunkTask.Hash, "error", getFailedTaskError) + return nil, ErrCoordinatorInternalFailure + } + for i := 0; i < len(proverTasks); i++ { + if proverTasks[i].ProverPublicKey == taskCtx.PublicKey || + taskCtx.ProverProviderType == uint8(coordinatorType.ProverProviderTypeExternal) && cutils.IsExternalProverNameMatch(proverTasks[i].ProverName, taskCtx.ProverName) { + log.Debug("get empty chunk, the prover already failed this task", "height", getTaskParameter.ProverHeight, "task ID", tmpChunkTask.Hash, "prover name", taskCtx.ProverName, "prover public key", taskCtx.PublicKey) + return nil, nil + } } }