IGNITE-28242 Add reproducer, add assertion for cp read lock

Author: sergey-chugunov-1985

modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java

@@ -457,6 +457,8 @@ protected GridDhtLocalPartition localPartition() {
         boolean deferred = false;
         GridCacheVersion ver0 = null;
 
+        assert !checkExpire || cctx.shared().database().checkpointLockIsHeldByThread();
+
         lockEntry();
 
         try {

modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/TxPutTxGetCheckpointerDeadlockTest.java

@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cluster.ClusterState;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.StopNodeFailureHandler;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointManager;
+import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointTimeoutLock;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.junit.Test;
+
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
+
+/**
+ * Test verifies there is no deadlock between GridCacheMapEntry.unswap() in tx put operation and a checkpointer requesting cp write lock
+ * with a parallel tx get operation.
+ * <p/>
+ * Root cause of the deadlock was wrong locking order when cp readlock is acquired under already locked GridCacheMapEntry instance
+ * by the first tx put op.
+ */
+public class TxPutTxGetCheckpointerDeadlockTest extends GridCommonAbstractTest {
+    /** */
+    private final AtomicBoolean deadlockDetected = new AtomicBoolean(false);
+
+    /** */
+    private final AtomicBoolean testFinished = new AtomicBoolean(false);
+
+    /** */
+    private static final String CP_WRITE_LOCK_SWITCHING_THREAD_NAME = "cp-write-lock-switching-runner";
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                )
+        );
+
+        cfg.setFailureHandler(new StopNodeFailureHandler());
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @SuppressWarnings({"deprecation"})
+    @Override protected void afterTest() throws Exception {
+        if (deadlockDetected.get()) {
+            Thread.getAllStackTraces().keySet().stream()
+                .filter(t -> t.getName().startsWith(CP_WRITE_LOCK_SWITCHING_THREAD_NAME))
+                .forEach(Thread::interrupt);
+        }
+
+        testFinished.set(true);
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * Tests for the absence of a deadlock between transactional cache operations and checkpointer write lock acquisition.
+     * <p>
+     * This test simulates a scenario where:
+     * <ul>
+     *   <li>One thread performs continuous transactional {@code put} operations on a cache entry.</li>
+     *   <li>Another thread performs continuous transactional {@code get} operations on the same entry, potentially
+     *       triggering unswapping of the entry under lock.</li>
+     *   <li>A third thread repeatedly acquires and releases the checkpoint write lock, simulating checkpointer activity.</li>
+     * </ul>
+     * </p>
+     * <p>
+     * The primary purpose is to verify that there is no deadlock caused by incorrect lock ordering,
+     * specifically when a transactional {@code put} holds a lock on a {@link org.apache.ignite.internal.processors.cache.GridCacheMapEntry}
+     * while attempting to acquire a checkpoint read lock, at the same time as the checkpointer
+     * tries to acquire checkpoint write lock.
+     * </p>
+     *
+     * @throws Exception If failed.
+     */
+    @Test
+    public void testDeadlock() throws Exception {
+        IgniteEx ignite = startGrid(0);
+
+        ignite.cluster().state(ClusterState.ACTIVE);
+
+        IgniteCache<Object, Object> cache = ignite
+            .getOrCreateCache(new CacheConfiguration<>("test").setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL));
+
+        AtomicInteger cnt = new AtomicInteger(-1);
+
+        GridTestUtils.runAsync(
+            () -> {
+                while (!testFinished.get()) {
+                    try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, READ_COMMITTED)) {
+                        cache.put(0, cnt.incrementAndGet());
+                        tx.commit();
+                    }
+                }
+            },
+            "write-tx-runner"
+        );
+
+        GridTestUtils.runAsync(
+            () -> {
+                while (!testFinished.get()) {
+                    try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, READ_COMMITTED)) {
+                        cache.get(0);
+                    }
+                }
+            },
+            "read-tx-runner"
+        );
+
+        GridTestUtils.runAsync(
+            () -> {
+                IgniteCacheDatabaseSharedManager db = ignite.context().cache().context().database();
+                CheckpointManager cpMgr = ((GridCacheDatabaseSharedManager)db).getCheckpointManager();
+                CheckpointTimeoutLock timeoutLock = cpMgr.checkpointTimeoutLock();
+                ReentrantReadWriteLock.WriteLock cpWriteLock = GridTestUtils.getFieldValue(timeoutLock,
+                    "checkpointReadWriteLock", "checkpointLock", "writeLock");
+
+                while (!testFinished.get()) {
+                    // an interruptible version of lock method is used to allow end the deadlock at the end of the test
+                    cpWriteLock.lockInterruptibly();
+                    cpWriteLock.unlock();
+                }
+            },
+            CP_WRITE_LOCK_SWITCHING_THREAD_NAME
+        );
+
+        GridTestUtils.runAsync(
+            () -> {
+                int prevVal = -1;
+
+                while (!testFinished.get()) {
+                    int curVal = cnt.get();
+
+                    if (curVal != -1 && curVal == prevVal) {
+                        deadlockDetected.set(true);
+
+                        return;
+                    }
+                    else {
+                        prevVal = curVal;
+
+                        doSleep(200);
+                    }
+                }
+            },
+            "progress-monitor"
+        );
+
+        assertTrue(GridTestUtils.waitForCondition(() -> cnt.get() != -1, 10_000L));
+        assertFalse("Unexpected deadlock detected", GridTestUtils.waitForCondition(deadlockDetected::get, 10_000L));
+    }
+}

modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite7.java

@@ -21,6 +21,7 @@
 import java.util.Collection;
 import java.util.List;
 import org.apache.ignite.internal.processors.cache.persistence.EagerTtlTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.TxPutTxGetCheckpointerDeadlockTest;
 import org.apache.ignite.internal.processors.cache.persistence.wal.WalRotatedIdPartRecordTest;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.DynamicSuite;
@@ -47,6 +48,8 @@ public static List<Class<?>> suite(Collection<Class> ignoredTests) {
 
         GridTestUtils.addTestIfNeeded(suite, WalRotatedIdPartRecordTest.class, ignoredTests);
 
+        GridTestUtils.addTestIfNeeded(suite, TxPutTxGetCheckpointerDeadlockTest.class, ignoredTests);
+
         return suite;
     }
 }