ci: containerize CI pipeline with pre-built Docker image

Add a BuildCIImage job that produces a content-hash-cached Docker image
(Ubuntu 22.04, JDK 8, conda, SBT, Spark, pre-cached datasets) published
to mmlsparkmcr.azurecr.io/synapseml/ci via Workload Identity Federation.

All test/build jobs now run inside this container, eliminating per-job
conda setup, CLI updates, and dataset downloads. Key changes:

- tools/docker/ci/Dockerfile: multi-stage image definition with pinned
  Miniconda, SHA256-verified libssl1.1, tightened file permissions (755)
- pipeline.yaml: container resource, BuildCIImage job, scoped compilation
  via PROJECT variable, SBT_OPTS tuning, free_disk.yml template,
  succeeded() gating on DatabricksE2E condition
- build.sbt: ForkJoinPool deadlock fix for parallel test execution
- TestBase.scala: 10-minute per-test hard timeout for all test suites
  (overridable in subclasses, e.g. DatabricksTestHelper uses longer)
- CodegenPlugin.scala: remove redundant root publishLocal in installPipPackage
- templates/free_disk.yml: disk cleanup helper

Note: FabricE2E intentionally excluded from containerization — it
requires Fabric SDK / sempy packages not in the CI container image.

Performance: ~59% wall-clock reduction (66m -> 27m), 33.5% total compute
savings (1228m -> 817m agent-minutes) across 59 test jobs.

Author: BrendanWalsh

.dockerignore

@@ -1,6 +1,10 @@
-target
-.idea
-.git
-tools/docker/Dockerfile.dev
-pipeline.yaml
-.dockerignore
+# Allowlist: only send files referenced by COPY instructions in Dockerfiles.
+# CI Dockerfile:  environment.yml, project/, build.sbt, sonatype.sbt
+# Demo Dockerfile: tools/docker/demo/init_notebook.py, docs/
+*
+!environment.yml
+!project/
+!build.sbt
+!sonatype.sbt
+!tools/docker/demo/init_notebook.py
+!docs/

.gitignore

@@ -95,3 +95,4 @@ condaenv.*.requirements.txt
 # Dev-loop artifacts
 .dev-loop/
 .dev-loop-artifacts/
+pipeline.yaml.bak

build.sbt

@@ -104,7 +104,12 @@ getDatasetsTask := {
   val f = new File(d, datasetName)
   if (!d.exists()) d.mkdirs()
   if (!f.exists()) {
-    FileUtils.copyURLToFile(datasetUrl, f)
+    val cached = new File(sys.env.getOrElse("DATASET_CACHE", "/opt/datasets"), datasetName)
+    if (cached.exists()) {
+      java.nio.file.Files.copy(cached.toPath, f.toPath)
+    } else {
+      FileUtils.copyURLToFile(datasetUrl, f)
+    }
     UnzipUtils.unzip(f, d)
   }
 }

core/src/test/scala/com/microsoft/azure/synapse/ml/core/test/base/TestBase.scala

@@ -22,7 +22,7 @@ import org.scalatest.time.{Seconds, Span}
 
 import java.io.File
 import java.nio.file.{Files, Path}
-import scala.concurrent._
+import scala.concurrent.blocking
 import scala.reflect.ClassTag
 
 trait SparkSessionManagement {
@@ -147,7 +147,18 @@ object TestBase extends SparkSessionManagement {
 
 }
 
-abstract class TestBase extends AnyFunSuite with BeforeAndAfterEachTestData with BeforeAndAfterAll {
+abstract class TestBase extends AnyFunSuite with BeforeAndAfterEachTestData with BeforeAndAfterAll with TimeLimits {
+
+  // Global per-test timeout (10 minutes). Override in subclass if needed.
+  val testTimeoutInSeconds: Int = 10 * 60
+
+  override def test(testName: String, testTags: Tag*)(testFun: => Any)(implicit pos: Position): Unit = {
+    super.test(testName, testTags: _*) {
+      failAfter(Span(testTimeoutInSeconds, Seconds)) {
+        testFun
+      }
+    }
+  }
 
   lazy val sparkProvider: SparkSessionManagement = TestBase