HIVE-29577: Support prefetching Tez snapshot jars into the Docker image (#6448)

Author: abstractdog

packaging/src/docker/Dockerfile

@@ -67,18 +67,22 @@ RUN tar -xzv \
     tar -xzv \
       --exclude="apache-tez-$TEZ_VERSION-bin/share" \
       -f /opt/apache-tez-$TEZ_VERSION-bin.tar.gz \
-      -C /opt
+      -C /opt; \
+    mkdir -p /opt/tez-snapshot;
 
 FROM eclipse-temurin:21-jdk-ubi9-minimal AS run
 
 ARG UID=1000
 ARG HADOOP_VERSION
 ARG HIVE_VERSION
 ARG TEZ_VERSION
+ARG TEZ_SNAPSHOT_VERSION=
+ARG TEZ_SNAPSHOT_REPO_URL=https://repository.apache.org/content/repositories/snapshots
+
 # Install dependencies
 RUN set -ex; \
     microdnf update -y; \
-    microdnf -y install procps gettext; \
+    microdnf -y install procps gettext wget xmlstarlet; \
     microdnf clean all; \
     useradd --no-create-home -s /sbin/nologin -c "" --uid $UID hive
 
@@ -93,6 +97,38 @@ ENV PATH=$HIVE_HOME/bin:$HADOOP_HOME/bin:$PATH
 COPY --from=env --chown=hive /opt/hadoop-$HADOOP_VERSION $HADOOP_HOME
 COPY --from=env --chown=hive /opt/apache-hive-$HIVE_VERSION-bin $HIVE_HOME
 COPY --from=env --chown=hive /opt/apache-tez-$TEZ_VERSION-bin $TEZ_HOME
+COPY --from=env --chown=hive /opt/tez-snapshot /opt/tez-snapshot
+
+# When TEZ_SNAPSHOT_VERSION is set, fetch Tez snapshot jars from the Maven snapshot repository
+# and place them under /opt/tez-snapshot. At runtime, entrypoint.sh symlinks these into
+# $HIVE_HOME/lib with a "0-" prefix so they sort first in bin/hive's classpath glob, ensuring
+# snapshot classes take precedence over the Tez release jars bundled with Hive.
+# Maven snapshot repositories use timestamped filenames (e.g. tez-api-1.0.0-20250101.jar),
+# so we fetch maven-metadata.xml first to resolve the exact filename before downloading the jar.
+RUN set -eux; \
+    mkdir -p /opt/tez-snapshot-download; \
+    if [[ -n "${TEZ_SNAPSHOT_VERSION}" ]]; then \
+      base_url="${TEZ_SNAPSHOT_REPO_URL}/org/apache/tez"; \
+      for artifact in tez-common tez-api tez-dag tez-mapreduce tez-runtime-internals tez-runtime-library; do \
+        version_url="${base_url}/${artifact}/${TEZ_SNAPSHOT_VERSION}"; \
+        metadata_url="${version_url}/maven-metadata.xml"; \
+        metadata_file="/opt/tez-snapshot-download/${artifact}-maven-metadata.xml"; \
+        echo "metadata_url=${metadata_url}"; \
+        wget -q "${metadata_url}" -O "${metadata_file}"; \
+        snapshot_value="$(xmlstarlet sel -t -v "string(/metadata/versioning/snapshotVersions/snapshotVersion[extension='jar' and not(classifier)]/value)" "${metadata_file}")"; \
+        test -n "${snapshot_value}"; \
+        jar_file="${artifact}-${snapshot_value}.jar"; \
+        jar_url="${version_url}/${jar_file}"; \
+        echo "jar_url=${jar_url}"; \
+        wget -q "${jar_url}" -O "/opt/tez-snapshot/${jar_file}"; \
+      done; \
+      echo "Downloaded Tez snapshot jars under /opt/tez-snapshot:"; \
+      ls -1 /opt/tez-snapshot/*.jar; \
+    else \
+      echo "TEZ_SNAPSHOT_VERSION not set. Skipping Tez snapshot download."; \
+    fi; \
+    rm -rf /opt/tez-snapshot-download
+
 
 COPY --chown=hive entrypoint.sh /
 COPY --chown=hive conf $HIVE_HOME/conf
@@ -103,7 +139,8 @@ RUN chmod +x /entrypoint.sh && \
     mkdir -p $HIVE_HOME/scratch && \
     chown hive $HIVE_HOME/scratch && \
     mkdir -p /home/hive/.beeline && \
-    chown hive /home/hive/.beeline
+    chown hive /home/hive/.beeline && \
+    chown -R hive /opt/tez-snapshot
 
 USER hive
 WORKDIR $HIVE_HOME

packaging/src/docker/build.sh

@@ -20,13 +20,15 @@ set -eux
 HIVE_VERSION=
 HADOOP_VERSION=
 TEZ_VERSION=
+TEZ_SNAPSHOT_VERSION=
 usage() {
     cat <<EOF 1>&2
-Usage: $0 [-h] [-hadoop <Hadoop version>] -tez <Tez release version> [-hive <Hive version>] [-repo <Docker repo>]
+Usage: $0 [-h] [-hadoop <Hadoop version>] -tez <Tez release version> [-tez-snapshot [<Maven snapshot version>]] [-hive <Hive version>] [-repo <Docker repo>]
 Build the Hive Docker image (reused for LLAP too)
 -help                Display help
 -hadoop              Build image with the specified Hadoop version (default: from Maven pom)
 -tez                 Required. Tez release tarball version (apache-tez-\$TEZ_VERSION-bin.tar.gz from archive)
+-tez-snapshot <ver>  Optional. When a snapshot version is given, fetch Tez Maven snapshot jars into the image. With no version, snapshot prefetch is skipped.
 -hive                Build image with the specified Hive version
 -repo                Docker repository
 EOF
@@ -48,6 +50,13 @@ while [ $# -gt 0 ]; do
       TEZ_VERSION=$1
       shift
       ;;
+    -tez-snapshot)
+      shift
+      if [ $# -gt 0 ] && [[ "$1" != -* ]]; then
+        TEZ_SNAPSHOT_VERSION=$1
+        shift
+      fi
+      ;;
     -hive)
       shift
       HIVE_VERSION=$1
@@ -135,6 +144,9 @@ DOCKER_BUILD_ARGS=(
   --build-arg "HADOOP_VERSION=$HADOOP_VERSION"
   --build-arg "TEZ_VERSION=$TEZ_VERSION"
 )
+if [ -n "$TEZ_SNAPSHOT_VERSION" ]; then
+  DOCKER_BUILD_ARGS+=(--build-arg "TEZ_SNAPSHOT_VERSION=$TEZ_SNAPSHOT_VERSION")
+fi
 
 docker build \
         "$WORK_DIR" \

packaging/src/docker/entrypoint.sh

@@ -147,8 +147,13 @@ function run_tezam {
   export HIVE_HOME="${HIVE_HOME:-/opt/hive}"
   export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HIVE_CONF_DIR}"
   export TEZ_CONF_DIR="${TEZ_CONF_DIR:-$HADOOP_CONF_DIR}"
+  : "${TEZ_SNAPSHOT_HOME:=/opt/tez-snapshot}"
+  if [[ ! -d "${TEZ_SNAPSHOT_HOME}" ]]; then
+    echo "Tez snapshot home not found at ${TEZ_SNAPSHOT_HOME}. Rebuild image to prefetch snapshot artifacts."
+    exit 1
+  fi
   # service_plugins_descriptor.json references org.apache.hadoop.hive.llap.tezplugins.* (hive-llap-tez, etc.)
-  tezam_cp="${HADOOP_CONF_DIR}:${TEZ_CONF_DIR}:${TEZ_HOME}/*:${TEZ_HOME}/lib/*:${HIVE_HOME}/lib/*:${HADOOP_HOME}/share/hadoop/common/*:${HADOOP_HOME}/share/hadoop/common/lib/*:${HADOOP_HOME}/share/hadoop/yarn/*:${HADOOP_HOME}/share/hadoop/yarn/lib/*:${HADOOP_HOME}/share/hadoop/hdfs/*:${HADOOP_HOME}/share/hadoop/hdfs/lib/*:${HADOOP_HOME}/share/hadoop/mapreduce/*:${HADOOP_HOME}/share/hadoop/mapreduce/lib/*:${HADOOP_CLASSPATH:-}"
+  tezam_cp="${HADOOP_CONF_DIR}:${TEZ_CONF_DIR}:${TEZ_SNAPSHOT_HOME}/*:${TEZ_HOME}/*:${TEZ_HOME}/lib/*:${HIVE_HOME}/lib/*:${HADOOP_HOME}/share/hadoop/common/*:${HADOOP_HOME}/share/hadoop/common/lib/*:${HADOOP_HOME}/share/hadoop/yarn/*:${HADOOP_HOME}/share/hadoop/yarn/lib/*:${HADOOP_HOME}/share/hadoop/hdfs/*:${HADOOP_HOME}/share/hadoop/hdfs/lib/*:${HADOOP_HOME}/share/hadoop/mapreduce/*:${HADOOP_HOME}/share/hadoop/mapreduce/lib/*:${HADOOP_CLASSPATH:-}"
 
   local java_bin
   local tezam_java_opts
@@ -178,6 +183,17 @@ if [[ "${SKIP_SCHEMA_INIT}" == "false" && ( "${SERVICE_NAME}" == "hiveserver2" |
 fi
 
 if [ "${SERVICE_NAME}" == "hiveserver2" ]; then
+  TEZ_SNAPSHOT_HOME="${TEZ_SNAPSHOT_HOME:-/opt/tez-snapshot}"
+  # bin/hive prepends all of $HIVE_HOME/lib/*.jar to $HADOOP_CLASSPATH, so any entry
+  # we put first in HADOOP_CLASSPATH ends up after all Hive lib jars.  To get snapshot jars
+  # truly first, symlink them into $HIVE_HOME/lib/ with a "0-" prefix: the for-loop glob in
+  # bin/hive processes jars alphabetically, and ASCII '0' (48) sorts before 'a' (97), so these
+  # symlinks become the very first jars on the classpath (right after the conf dir entries).
+  if ls "${TEZ_SNAPSHOT_HOME}"/*.jar 1>/dev/null 2>&1; then
+    for snap_jar in "${TEZ_SNAPSHOT_HOME}"/*.jar; do
+      ln -sf "$snap_jar" "${HIVE_HOME}/lib/0-$(basename "$snap_jar")"
+    done
+  fi
   export HADOOP_CLASSPATH="$TEZ_HOME/*:$TEZ_HOME/lib/*:$HADOOP_CLASSPATH"
   exec "$HIVE_HOME/bin/hive" --skiphadoopversion --skiphbasecp --service "$SERVICE_NAME"
 elif [ "${SERVICE_NAME}" == "metastore" ]; then