[cuebot] Recover lost dependencies (#2219)

Add a scheduled jobs to run every 10 minutes to search for dependencies
that should have been cleared and are still active.

Cuebot's dependency system relies solely on the FrameCompleteReport
being successfully processed, and with no recover logic, there's a great
risk of dangling dependencies, that usually require manual intervention
to clear dependencies and get frames rolling.

**Attention**
Three new properties have been added to opencue.properties, and one of
them (`maintenance.stuck_dependency_recovery_interval_ms`) is required:

```
# Enable/disable the stuck dependency recovery maintenance task.
maintenance.stuck_dependency_recovery_enabled=true

# Interval in milliseconds for stuck dependency recovery task.
# Default is 10 minutes (600000 ms). This task finds and fixes frames
# stuck in DEPEND state due to transient failures during dependency satisfaction.
maintenance.stuck_dependency_recovery_interval_ms=600000

# Maximum number of depends/frames to process per recovery run.
maintenance.stuck_dependency_recovery_batch_size=1000
```


### LLM Disclosure

Claude Opus model was used to brainstorm this PR, optimize queries and
review the work. All content has been fully reviewed and tested by the
author (me).

Author: DiegoTavares

VERSION.in

@@ -1 +1 @@
-1.17
+1.18

cuebot/src/main/java/com/imageworks/spcue/MaintenanceTask.java

@@ -49,5 +49,10 @@ public enum MaintenanceTask {
     /**
      * Lock the subscription recalculation task.
      */
-    LOCK_SUBSCRIPTION_RECALCULATION
+    LOCK_SUBSCRIPTION_RECALCULATION,
+
+    /**
+     * Lock the stuck dependency recovery task.
+     */
+    LOCK_STUCK_DEPENDENCY_RECOVERY
 }

cuebot/src/main/java/com/imageworks/spcue/dao/MaintenanceDao.java

@@ -15,6 +15,8 @@
 
 package com.imageworks.spcue.dao;
 
+import java.util.List;
+
 import com.imageworks.spcue.MaintenanceTask;
 
 /**
@@ -63,4 +65,24 @@ public interface MaintenanceDao {
      */
     void recalculateSubscriptions();
 
+    /**
+     * Finds active depends whose depended-on entity (job, layer, or frame) is already complete.
+     * These are stale depends that should have been satisfied but weren't due to transient
+     * failures.
+     *
+     * @param limit maximum number of depend IDs to return
+     * @return list of pk_depend values for stale active depends
+     */
+    List<String> findStaleDependIds(int limit);
+
+    /**
+     * Fixes frames stuck in DEPEND state by setting int_depend_count to 0 for frames that have no
+     * active depends targeting them. The DB trigger update_frame_dep_to_wait handles the DEPEND to
+     * WAITING state transition automatically.
+     *
+     * @param limit maximum number of frames to fix per invocation
+     * @return number of frames fixed
+     */
+    int fixStuckDependCounts(int limit);
+
 }

cuebot/src/main/java/com/imageworks/spcue/dao/postgres/MaintenanceDaoJdbc.java

@@ -15,6 +15,8 @@
 
 package com.imageworks.spcue.dao.postgres;
 
+import java.util.List;
+
 import org.springframework.jdbc.core.support.JdbcDaoSupport;
 
 import com.imageworks.spcue.MaintenanceTask;
@@ -62,4 +64,95 @@ public void unlockTask(MaintenanceTask task) {
     public void recalculateSubscriptions() {
         getJdbcTemplate().execute(RECALCULATE_SUBS);
     }
+
+    // spotless:off
+    /**
+     * Finds active, non-composite depends whose depended-upon entity has already completed
+     * successfully. Composite depends (b_composite = true) are parent depends that are
+     * automatically satisfied when all their child depends are satisfied, so they should
+     * not be resolved directly.
+     *
+     * Section 1: Job-level depends (JOB_ON_JOB, LAYER_ON_JOB, FRAME_ON_JOB) where
+     *   every frame in the depended-upon job has SUCCEEDED or been EATEN.
+     * Section 2: Layer-level depends (JOB_ON_LAYER, LAYER_ON_LAYER, FRAME_ON_LAYER) where
+     *   every frame in the depended-upon layer has SUCCEEDED or been EATEN.
+     * Section 3: Frame-level depends (JOB_ON_FRAME, LAYER_ON_FRAME, FRAME_ON_FRAME) where
+     *   the specific depended-upon frame has SUCCEEDED or been EATEN.
+     */
+    private static final String FIND_STALE_DEPEND_IDS =
+            // Section 1: Job-level depends where all frames in the job succeeded or were eaten.
+            // Pre-filter on job.str_state = 'FINISHED' leverages the i_job_str_state index to
+            // skip depends on active jobs. A FINISHED job may still have DEAD frames, so we
+            // also verify frame states via NOT EXISTS.
+            "SELECT pk_depend FROM depend d "
+            + "JOIN job j ON j.pk_job = d.pk_job_depend_on AND j.str_state = 'FINISHED' "
+            + "WHERE d.b_active = true "
+            + "AND d.b_composite = false "
+            + "AND d.pk_layer_depend_on IS NULL "
+            + "AND d.pk_frame_depend_on IS NULL "
+            + "AND d.str_type IN ('JOB_ON_JOB', 'LAYER_ON_JOB', 'FRAME_ON_JOB') "
+            + "AND NOT EXISTS ("
+                + "SELECT 1 FROM frame f "
+                + "WHERE f.pk_job = d.pk_job_depend_on "
+                + "AND f.str_state NOT IN ('SUCCEEDED', 'EATEN')) "
+
+            + "UNION ALL "
+
+            // Section 2: Layer-level depends where all frames in the layer succeeded or were eaten.
+            + "SELECT pk_depend FROM depend d "
+            + "WHERE d.b_active = true "
+            + "AND d.b_composite = false "
+            + "AND d.pk_layer_depend_on IS NOT NULL "
+            + "AND d.str_type IN ('JOB_ON_LAYER', 'LAYER_ON_LAYER', 'FRAME_ON_LAYER') "
+            + "AND NOT EXISTS ("
+                + "SELECT 1 FROM frame f "
+                + "WHERE f.pk_layer = d.pk_layer_depend_on "
+                + "AND f.str_state NOT IN ('SUCCEEDED', 'EATEN')) "
+
+            + "UNION ALL "
+
+            // Section 3: Frame-level depends where the specific frame succeeded or was eaten.
+            + "SELECT pk_depend FROM depend d "
+            + "WHERE d.b_active = true "
+            + "AND d.b_composite = false "
+            + "AND d.pk_frame_depend_on IS NOT NULL "
+            + "AND d.str_type IN ('JOB_ON_FRAME', 'LAYER_ON_FRAME', 'FRAME_ON_FRAME') "
+            + "AND EXISTS ("
+                + "SELECT 1 FROM frame f "
+                + "WHERE f.pk_frame = d.pk_frame_depend_on "
+                + "AND f.str_state IN ('SUCCEEDED', 'EATEN')) "
+
+            + "LIMIT ?";
+    // spotless:on
+
+    @Override
+    public List<String> findStaleDependIds(int limit) {
+        return getJdbcTemplate().queryForList(FIND_STALE_DEPEND_IDS, String.class, limit);
+    }
+
+    // spotless:off
+    private static final String FIX_STUCK_DEPEND_COUNTS =
+            "UPDATE frame SET int_depend_count = 0 "
+            + "WHERE pk_frame IN ("
+                + "SELECT f.pk_frame FROM frame f "
+                + "WHERE f.str_state = 'DEPEND' "
+                + "AND f.int_depend_count > 0 "
+                + "AND NOT EXISTS ("
+                    + "SELECT 1 FROM depend d "
+                    + "WHERE d.b_active = true "
+                    + "AND d.pk_job_depend_er = f.pk_job "
+                    + "AND ("
+                        + "d.pk_frame_depend_er = f.pk_frame "
+                        + "OR (d.pk_layer_depend_er = f.pk_layer AND d.pk_frame_depend_er IS NULL) "
+                        + "OR (d.pk_layer_depend_er IS NULL AND d.pk_frame_depend_er IS NULL)"
+                    + ")"
+                + ") "
+                + "LIMIT ?"
+            + ")";
+    // spotless:on
+
+    @Override
+    public int fixStuckDependCounts(int limit) {
+        return getJdbcTemplate().update(FIX_STUCK_DEPEND_COUNTS, limit);
+    }
 }

cuebot/src/main/java/com/imageworks/spcue/service/MaintenanceManagerSupport.java

@@ -27,6 +27,7 @@
 
 import com.imageworks.spcue.FrameDetail;
 import com.imageworks.spcue.FrameInterface;
+import com.imageworks.spcue.LightweightDependency;
 import com.imageworks.spcue.MaintenanceTask;
 import com.imageworks.spcue.PointDetail;
 import com.imageworks.spcue.VirtualProc;
@@ -63,6 +64,8 @@ public class MaintenanceManagerSupport {
 
     private DepartmentManager departmentManager;
 
+    private DependManager dependManager;
+
     private static final long WAIT_FOR_HOST_REPORTS_MS = 600000;
 
     private static final int CHECKPOINT_MAX_WAIT_SEC = 300;
@@ -243,6 +246,61 @@ public void recalculateSubscriptions() {
         }
     }
 
+    /**
+     * Recovers frames stuck in DEPEND state due to transient failures during dependency
+     * satisfaction. Runs in two phases:
+     *
+     * Phase 1: Finds active depends whose depended-on entity is already complete and satisfies them
+     * through the normal code path (handles composite depends and publishes events). Each
+     * {@code satisfyDepend} call runs in its own transaction via {@code DependManagerService}'s
+     * {@code @Transactional}, providing per-depend isolation.
+     *
+     * Phase 2: Fixes frames still stuck in DEPEND by resetting int_depend_count to 0 where no
+     * active depends target them. The DB trigger {@code update_frame_dep_to_wait} handles the
+     * DEPEND to WAITING transition. This UPDATE auto-commits as a single statement transaction.
+     */
+    public void recoverStuckDependencies() {
+        if (!env.getProperty("maintenance.stuck_dependency_recovery_enabled", Boolean.class,
+                true)) {
+            return;
+        }
+
+        // Use a MaintenanceTask lock to prevent multiple instances from racing each other
+        if (!maintenanceDao.lockTask(MaintenanceTask.LOCK_STUCK_DEPENDENCY_RECOVERY)) {
+            return;
+        }
+        try {
+            int batchSize = env.getProperty("maintenance.stuck_dependency_recovery_batch_size",
+                    Integer.class, 1000);
+
+            // Phase 1: Satisfy stale active depends through normal code path
+            List<String> staleDependIds = maintenanceDao.findStaleDependIds(batchSize);
+            int satisfiedCount = 0;
+            for (String dependId : staleDependIds) {
+                try {
+                    LightweightDependency depend = dependManager.getDepend(dependId);
+                    dependManager.satisfyDepend(depend);
+                    satisfiedCount++;
+                } catch (Exception e) {
+                    logger.warn("failed to satisfy stale depend " + dependId + ": " + e);
+                }
+            }
+            if (satisfiedCount > 0) {
+                logger.info("recovered " + satisfiedCount + " stale active depends");
+            }
+
+            // Phase 2: Fix count mismatches via SQL
+            int fixedFrames = maintenanceDao.fixStuckDependCounts(batchSize);
+            if (fixedFrames > 0) {
+                logger.info("fixed depend counts for " + fixedFrames + " stuck frames");
+            }
+        } catch (Exception e) {
+            logger.warn("failed to recover stuck dependencies: " + e);
+        } finally {
+            maintenanceDao.unlockTask(MaintenanceTask.LOCK_STUCK_DEPENDENCY_RECOVERY);
+        }
+    }
+
     public FrameDao getFrameDao() {
         return frameDao;
     }
@@ -303,4 +361,12 @@ public void setJobManager(JobManager jobManager) {
         this.jobManager = jobManager;
     }
 
+    public DependManager getDependManager() {
+        return dependManager;
+    }
+
+    public void setDependManager(DependManager dependManager) {
+        this.dependManager = dependManager;
+    }
+
 }

cuebot/src/main/resources/conf/ddl/postgres/migrations/V38__Add_stuck_dependency_recovery_task_lock.sql

@@ -0,0 +1,6 @@
+-- Add task_lock entry for stuck dependency recovery maintenance task.
+-- This task periodically detects and fixes frames stuck in DEPEND state
+-- due to transient failures during dependency satisfaction.
+
+INSERT INTO task_lock (pk_task_lock, str_name, int_lock, int_timeout)
+VALUES ('00000000-0000-0000-0000-000000000007', 'LOCK_STUCK_DEPENDENCY_RECOVERY', 0, 600);

cuebot/src/main/resources/conf/ddl/postgres/seed_data.sql

@@ -84,3 +84,5 @@ Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('0000
 Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000005','LOCK_TASK_UPDATE',1240618998852,3600);
 
 Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000006','LOCK_SUBSCRIPTION_RECALCULATION',0,7200);
+
+Insert into TASK_LOCK (PK_TASK_LOCK,STR_NAME,INT_LOCK,INT_TIMEOUT) values ('00000000-0000-0000-0000-000000000007','LOCK_STUCK_DEPENDENCY_RECOVERY',0,600);

cuebot/src/main/resources/conf/spring/applicationContext-service.xml

@@ -422,6 +422,7 @@
     <property name="historicalSupport" ref="historicalSupport" />
     <property name="departmentManager" ref="departmentManager" />
     <property name="jobManager" ref="jobManager" />
+    <property name="dependManager" ref="dependManager" />
   </bean>
 
   <bean id="hardwareCheck" class="org.springframework.scheduling.quartz.MethodInvokingJobDetailFactoryBean">
@@ -604,6 +605,19 @@
     <property name="repeatInterval" value="${maintenance.subscription_recalculation_interval_ms}" />
   </bean>
 
+  <bean id="stuckDependencyRecovery" class="org.springframework.scheduling.quartz.MethodInvokingJobDetailFactoryBean">
+    <property name="targetObject" ref="maintenanceManager" />
+    <property name="targetMethod" value="recoverStuckDependencies" />
+  </bean>
+
+  <bean id="stuckDependencyRecoveryTrigger" class="org.springframework.scheduling.quartz.SimpleTriggerFactoryBean">
+    <property name="jobDetail" ref="stuckDependencyRecovery" />
+    <!-- 10 minute start delay to let the cue get up and running -->
+    <property name="startDelay" value="600000" />
+    <!-- repeat every 10 minutes by default (configurable via maintenance.stuck_dependency_recovery_interval_ms) -->
+    <property name="repeatInterval" value="${maintenance.stuck_dependency_recovery_interval_ms}" />
+  </bean>
+
   <bean class="org.springframework.scheduling.quartz.SchedulerFactoryBean" destroy-method="destroy">
    <property name="waitForJobsToCompleteOnShutdown"><value>false</value></property>
     <property name="triggers">
@@ -621,6 +635,7 @@
         <ref bean="updateShowsStatusTrigger" />
         <ref bean="collectPrometheusMetricsTrigger" />
         <ref bean="subscriptionRecalculationTrigger" />
+        <ref bean="stuckDependencyRecoveryTrigger" />
       </list>
     </property>
   </bean>

cuebot/src/main/resources/opencue.properties

@@ -187,6 +187,17 @@ maintenance.auto_delete_down_hosts=false
 # accountability issues that can occur at large scale.
 maintenance.subscription_recalculation_interval_ms=7200000
 
+# Enable/disable the stuck dependency recovery maintenance task.
+maintenance.stuck_dependency_recovery_enabled=true
+
+# Interval in milliseconds for stuck dependency recovery task.
+# Default is 10 minutes (600000 ms). This task finds and fixes frames
+# stuck in DEPEND state due to transient failures during dependency satisfaction.
+maintenance.stuck_dependency_recovery_interval_ms=600000
+
+# Maximum number of depends/frames to process per recovery run.
+maintenance.stuck_dependency_recovery_batch_size=1000
+
 # Set hostname/IP of the smtp host. Will be used for mailing
 smtp_host=smtp
 

cuebot/src/test/resources/opencue.properties

@@ -79,3 +79,8 @@ log.loki.url = http://localhost/loki/api
 
 # Interval in milliseconds for subscription recalculation task.
 maintenance.subscription_recalculation_interval_ms=7200000
+
+# Interval in milliseconds for stuck dependency recovery task.
+# Default is 10 minutes (600000 ms). This task finds and fixes frames
+# stuck in DEPEND state due to transient failures during dependency satisfaction.
+maintenance.stuck_dependency_recovery_interval_ms=600000