Repository: apache/hive
Pull request #5649: HIVE-28770: Fix Session Directories Persisting After HiveServer2 Shutdown
Author: login: Indhumathi27
Labels: name: tests passed
Created: 2025-02-20T19:22:03Z | Merged: 2025-04-02T08:36:37Z

### What changes were proposed in this pull request?
Ensures session related directories is properly cleaned up when HiveServer2 (HS2) shuts down, by adding a shutdown hook to call close() method, which ensures cleaning session related directories.

### Why are the changes needed?
Session related directories persist after HS2 shutdown, leading to disk bloat and performance degradation. This fix prevents unnecessary resource accumulation and improves cleanup.

### Does this PR introduce _any_ user-facing change?
No functional changes for users; it only enhances internal resource management. Temporary directories will now be cleaned up automatically.

### Is the change a dependency upgrade?
No, this PR does not introduce any dependency upgrades.

### How was this patch tested?
Manually verified that session directories on /tmp and /tmp/{user_dir}/ are created and removed correctly. 


Diff:
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java
index 1a0531bd7a93..ad1ac2d9a6aa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java
@@ -20,8 +20,6 @@
 
 import java.io.IOException;
 
-import org.apache.hadoop.hive.common.JavaUtils;
-import org.apache.hadoop.hive.common.ValidWriteIdList;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.hooks.HookContext;
@@ -92,6 +90,14 @@ private static void runOnDriverInternal(String query, HiveConf conf, SessionStat
     }
   }
 
+  public static SessionState setUpAndStartSessionState(HiveConf conf) {
+    return setUpAndStartSessionState(conf, null);
+  }
+
+  public static SessionState setUpAndStartSessionState(HiveConf conf, String user) {
+    return setUpSessionState(conf, user, true);
+  }
+
   public static SessionState setUpSessionState(HiveConf conf, String user, boolean doStart) {
     SessionState sessionState = SessionState.get();
     if (sessionState == null) {
@@ -104,8 +110,7 @@ public static SessionState setUpSessionState(HiveConf conf, String user, boolean
         SessionState.start(sessionState);
       }
       SessionState.setCurrentSessionState(sessionState);
-    }
-    else {
+    } else {
       sessionState.setConf(conf);
     }
     return sessionState;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillTriggerActionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillTriggerActionHandler.java
index cb279987a06e..70241345bfab 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillTriggerActionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillTriggerActionHandler.java
@@ -20,9 +20,10 @@
 import java.util.Map;
 
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.DriverUtils;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.session.KillQuery;
-import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.wm.Action;
 import org.apache.hadoop.hive.ql.wm.Trigger;
 import org.apache.hadoop.hive.ql.wm.TriggerActionHandler;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -34,32 +35,33 @@
  */
 public class KillTriggerActionHandler implements TriggerActionHandler<TezSessionState> {
   private static final Logger LOG = LoggerFactory.getLogger(KillTriggerActionHandler.class);
+  private final HiveConf conf;
+
+  public KillTriggerActionHandler() {
+      this.conf = new HiveConf();
+  }
 
   @Override
   public void applyAction(final Map<TezSessionState, Trigger> queriesViolated) {
     for (Map.Entry<TezSessionState, Trigger> entry : queriesViolated.entrySet()) {
-      switch (entry.getValue().getAction().getType()) {
-      case KILL_QUERY:
-        TezSessionState sessionState = entry.getKey();
-        String queryId = sessionState.getWmContext().getQueryId();
-        try {
-          UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-          SessionState ss = new SessionState(new HiveConf(), ugi.getShortUserName());
-          ss.setIsHiveServerQuery(true);
-          SessionState.start(ss);
-          KillQuery killQuery = sessionState.getKillQuery();
-          // if kill query is null then session might have been released to pool or closed already
-          if (killQuery != null) {
-            sessionState.getKillQuery().killQuery(queryId, entry.getValue().getViolationMsg(),
-                      sessionState.getConf());
-          }
-        } catch (HiveException|IOException e) {
-          LOG.warn("Unable to kill query {} for trigger violation", queryId);
+        if (entry.getValue().getAction().getType() == Action.Type.KILL_QUERY) {
+            TezSessionState sessionState = entry.getKey();
+            String queryId = sessionState.getWmContext().getQueryId();
+            try {
+                UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+                DriverUtils.setUpAndStartSessionState(conf, ugi.getShortUserName());
+                KillQuery killQuery = sessionState.getKillQuery();
+                // if kill query is null then session might have been released to pool or closed already
+                if (killQuery != null) {
+                    sessionState.getKillQuery().killQuery(queryId, entry.getValue().getViolationMsg(),
+                            sessionState.getConf());
+                }
+            } catch (HiveException | IOException e) {
+                LOG.warn("Unable to kill query {} for trigger violation", queryId);
+            }
+        } else {
+            throw new RuntimeException("Unsupported action: " + entry.getValue());
         }
-        break;
-      default:
-        throw new RuntimeException("Unsupported action: " + entry.getValue());
-      }
     }
   }
 }
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManager.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManager.java
index 2c71296772fd..c8c40e386172 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManager.java
@@ -67,6 +67,7 @@
 import org.apache.hadoop.hive.metastore.api.WMPool;
 import org.apache.hadoop.hive.metastore.api.WMPoolTrigger;
 import org.apache.hadoop.hive.metastore.api.WMTrigger;
+import org.apache.hadoop.hive.ql.DriverUtils;
 import org.apache.hadoop.hive.ql.exec.tez.AmPluginNode.AmPluginInfo;
 import org.apache.hadoop.hive.ql.exec.tez.TezSessionState.HiveResources;
 import org.apache.hadoop.hive.ql.exec.tez.UserPoolMapping.MappingInput;
@@ -486,9 +487,7 @@ private void scheduleWork(WmThreadSyncWork context) {
             LOG.info("Invoking KillQuery for " + queryId + ": " + reason);
             try {
               UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-              SessionState ss = new SessionState(new HiveConf(), ugi.getShortUserName());
-              ss.setIsHiveServerQuery(true);
-              SessionState.start(ss);
+              DriverUtils.setUpAndStartSessionState(conf, ugi.getShortUserName());
               kq.killQuery(queryId, reason, toKill.getConf());
               addKillQueryResult(toKill, true);
               killCtx.killSessionFuture.set(true);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
index 9e1320e62ea1..f2379fb09f52 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
@@ -54,6 +54,7 @@
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.DriverUtils;
 import org.apache.hadoop.hive.ql.exec.ColumnInfo;
 import org.apache.hadoop.hive.ql.log.PerfLogger;
 import org.apache.hadoop.hive.ql.optimizer.calcite.HiveTypeSystemImpl;
@@ -167,12 +168,10 @@ private Loader(Hive db) {
 
     @Override
     public void run() {
-      SessionState ss = new SessionState(db.getConf());
-      ss.setIsHiveServerQuery(true); // All is served from HS2, we do not need e.g. Tez sessions
-      SessionState.start(ss);
       PerfLogger perfLogger = SessionState.getPerfLogger();
-      perfLogger.perfLogBegin(CLASS_NAME, PerfLogger.MATERIALIZED_VIEWS_REGISTRY_REFRESH);
       try {
+        DriverUtils.setUpAndStartSessionState(db.getConf());
+        perfLogger.perfLogBegin(CLASS_NAME, PerfLogger.MATERIALIZED_VIEWS_REGISTRY_REFRESH);
         if (initialized.get()) {
           for (Table mvTable : db.getAllMaterializedViewObjectsForRewriting()) {
             RelOptMaterialization existingMV = getRewritingMaterializedView(
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 5df2885a3a32..5801ffba4e77 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -44,7 +44,12 @@
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.CancellationException;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.ReentrantLock;
 
@@ -110,6 +115,7 @@
 import org.apache.hadoop.hive.shims.Utils;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hive.common.util.ShutdownHookManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -124,7 +130,7 @@
  * from any point in the code to interact with the user and to retrieve
  * configuration information
  */
-public class SessionState implements ISessionAuthState{
+public class SessionState implements ISessionAuthState {
   private static final Logger LOG = LoggerFactory.getLogger(SessionState.class);
 
   public static final String TMP_PREFIX = "_tmp_space.db";
@@ -341,6 +347,14 @@ public enum AuthorizationMode{V1, V2};
    */
   private boolean compaction = false;
 
+  // A thread-safe set to hold all active session states
+  private static final Set<SessionState> sessionStates = ConcurrentHashMap.newKeySet();
+
+  // Static block to add a single shutdown hook to clean up all session states
+  static {
+    ShutdownHookManager.addShutdownHook(SessionState::cleanUpAllSessionStates);
+  }
+
   public QueryState getQueryState(String queryId) {
     return queryStateMap.get(queryId);
   }
@@ -667,6 +681,10 @@ public static void detachSession() {
    */
   public static SessionState start(SessionState startSs) {
     start(startSs, false, null);
+
+    // Register the session state in the centralized set
+    sessionStates.add(startSs);
+
     return startSs;
   }
 
@@ -1906,6 +1924,8 @@ public void setCurrentCatalog(String currentCatalog) {
   }
 
   public void close() throws IOException {
+    // de-register session state
+    sessionStates.remove(this);
     for (Closeable cleanupItem : cleanupItems) {
       try {
         cleanupItem.close();
@@ -1969,6 +1989,32 @@ public void close() throws IOException {
     dynamicVars.clear();
   }
 
+  private static void cleanUpAllSessionStates() {
+    ExecutorService cleanupExecutor = Executors.newCachedThreadPool();
+    try {
+      CompletableFuture<Void> allCleanupTasks = CompletableFuture.allOf(sessionStates.stream()
+              .map(sessionState -> CompletableFuture.runAsync(() -> {
+                  try {
+                      LOG.info("Closing session state: {}", sessionState.getSessionId());
+                      sessionState.close();
+                  } catch (IOException e) {
+                      throw new CompletionException(e);
+                  }
+              }, cleanupExecutor).exceptionally(e -> {
+                  LOG.error("Problem closing session state", e);
+                  return null;
+              })).toArray(CompletableFuture[]::new));
+      try {
+        allCleanupTasks.get(60, TimeUnit.SECONDS);
+      } catch (Exception e) {
+        LOG.error("Failed to close all session states", e);
+      }
+    } finally {
+      // shutdown cleanup executor after all tasks finished/timeout
+      cleanupExecutor.shutdownNow();
+    }
+  }
+
   private void clearReflectionUtilsCache() {
     Method clearCacheMethod;
     try {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java
index 585fc114189b..4638451f3b55 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java
@@ -77,7 +77,7 @@ protected SessionState setupQueryCompactionSession(HiveConf conf, CompactionInfo
     Util.overrideConfProps(conf, compactionInfo, tblProperties);
     
     String user = compactionInfo.runAs;
-    SessionState sessionState = DriverUtils.setUpSessionState(conf, user, true);
+    SessionState sessionState = DriverUtils.setUpAndStartSessionState(conf, user);
     sessionState.setCompaction(true);
     
     return sessionState;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/StatsUpdater.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/StatsUpdater.java
index 3cd80955e113..a3b940e74d57 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/StatsUpdater.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/StatsUpdater.java
@@ -18,7 +18,6 @@
 package org.apache.hadoop.hive.ql.txn.compactor;
 
 import org.apache.hadoop.hive.common.ValidTxnList;
-import org.apache.hadoop.hive.common.ValidTxnWriteIdList;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.IMetaStoreClient;
 import org.apache.hadoop.hive.metastore.Warehouse;
@@ -91,7 +90,7 @@ public void gatherStats(CompactionInfo ci, HiveConf hiveConf,
             if (compactionQueueName != null && compactionQueueName.length() > 0) {
                 conf.set(TezConfiguration.TEZ_QUEUE_NAME, compactionQueueName);
             }
-            SessionState sessionState = DriverUtils.setUpSessionState(conf, userName, true);
+            SessionState sessionState = DriverUtils.setUpAndStartSessionState(conf, userName);
             DriverUtils.runOnDriver(conf, sessionState, sb.toString());
         } catch (Throwable t) {
             LOG.error(ci + ": gatherStats(" + ci.dbname + "," + ci.tableName + "," + ci.partName +
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java
index b97a8fecf625..66c4d1bbc532 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUpdaterThread.java
@@ -91,7 +91,7 @@ public void setUp() throws Exception {
     if (!(new File(getTestDataDir()).mkdirs())) {
       throw new RuntimeException("Could not create " + getTestDataDir());
     }
-    this.ss = DriverUtils.setUpSessionState(hiveConf, "hive", true);
+    this.ss = DriverUtils.setUpAndStartSessionState(hiveConf, "hive");
     cleanUp();
   }
 
diff --git a/service/src/java/org/apache/hive/service/cli/CLIService.java b/service/src/java/org/apache/hive/service/cli/CLIService.java
index a061536f857d..eb53607771b1 100644
--- a/service/src/java/org/apache/hive/service/cli/CLIService.java
+++ b/service/src/java/org/apache/hive/service/cli/CLIService.java
@@ -32,6 +32,7 @@
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.ql.DriverUtils;
 import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
 import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -129,9 +130,7 @@ private void applyAuthorizationConfigPolicy(HiveConf newHiveConf) throws HiveExc
       MetaException {
     // authorization setup using SessionState should be revisited eventually, as
     // authorization and authentication are not session specific settings
-    SessionState ss = new SessionState(newHiveConf);
-    ss.setIsHiveServerQuery(true);
-    SessionState.start(ss);
+    SessionState ss = DriverUtils.setUpAndStartSessionState(newHiveConf);
     ss.applyAuthorizationPolicy();
   }
 


Reviews:
user: login: okumin
body: The issue is very well specified in the JIRA ticket. Thanks.
https://issues.apache.org/jira/browse/HIVE-28770

I asked you some questions because I'm still new to the parts that this PR changes. The intention and the fundamental approach look good to me
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-02-28T14:28:10Z
user: login: Indhumathi27
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-03T16:05:07Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-04T10:49:16Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-04T10:50:05Z
user: login: Indhumathi27
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T11:39:06Z
user: login: Indhumathi27
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T13:07:12Z
user: login: Indhumathi27
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-04T13:08:52Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-05T05:51:14Z
user: login: Indhumathi27
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-11T16:29:39Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-11T17:41:13Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-11T17:58:54Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-11T18:02:49Z
user: login: kasakrisz
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T11:28:51Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T12:20:50Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T12:23:43Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T12:30:15Z
user: login: Indhumathi27
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T12:36:44Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T12:42:42Z
user: login: Indhumathi27
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T12:44:46Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T12:47:24Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T12:48:26Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T12:49:36Z
user: login: abstractdog
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T13:47:04Z
user: login: abstractdog
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T14:06:38Z
user: login: abstractdog
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-13T14:09:57Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T18:53:12Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T18:55:30Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T18:58:33Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T18:59:27Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T19:00:31Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T19:02:42Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T19:05:03Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-13T19:12:58Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-17T15:46:03Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-17T15:47:10Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-17T15:52:03Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T08:37:39Z
user: login: deniskuzZ
body: Thanks, @Indhumathi27 for the fix! The changes look good to me, 1 minor comment.
+1, pending tests
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-18T08:39:36Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T08:42:29Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T08:50:23Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T08:51:11Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T09:19:44Z
user: login: Indhumathi27
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T09:21:44Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T09:23:02Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T09:23:49Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T09:32:23Z
user: login: Indhumathi27
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T10:21:29Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T12:48:28Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T13:05:36Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T16:00:36Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-27T19:02:01Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-27T19:06:41Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-27T19:08:30Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-27T19:09:55Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-27T19:49:25Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-27T20:43:22Z
user: login: Indhumathi27
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-28T07:45:10Z
user: login: Indhumathi27
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-28T07:57:04Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-28T08:59:34Z
user: login: Indhumathi27
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-28T11:43:18Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-28T16:21:15Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-28T16:50:45Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-28T16:54:58Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-28T17:18:27Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-28T17:21:00Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-28T17:36:29Z
user: login: deniskuzZ
body: LGTM
@kasakrisz, @abstractdog can we merge?
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-31T12:10:37Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-31T12:37:34Z
user: login: kasakrisz
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-01T06:45:09Z
user: login: deniskuzZ
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-01T07:40:52Z

Review comments:
diff_hunk: @@ -204,11 +205,21 @@ public void run() {
         } else {
           LOG.error("Problem connecting to the metastore when initializing the view registry", e);
         }
+      } finally {
+          cleanUpResources(ss);
path: ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
user: login: okumin
body: I'm curious about what would happen if we closed `ss` using `SessionState#close`.
created_at: 2025-02-28T14:11:46Z
author_association: CONTRIBUTOR
diff_hunk: @@ -204,11 +205,21 @@ public void run() {
         } else {
           LOG.error("Problem connecting to the metastore when initializing the view registry", e);
         }
+      } finally {
+          cleanUpResources(ss);
       }
       perfLogger.perfLogEnd(CLASS_NAME, PerfLogger.MATERIALIZED_VIEWS_REGISTRY_REFRESH);
     }
   }
 
+  // Method to clean up the downloaded resources
+  private void cleanUpResources(SessionState sessionState) {
+    if (sessionState != null) {
path: ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
user: login: okumin
body: I believe this is always non-null.
created_at: 2025-02-28T14:12:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -133,6 +135,13 @@ private void applyAuthorizationConfigPolicy(HiveConf newHiveConf) throws HiveExc
     ss.setIsHiveServerQuery(true);
     SessionState.start(ss);
     ss.applyAuthorizationPolicy();
path: service/src/java/org/apache/hive/service/cli/CLIService.java
user: login: okumin
body: I slightly feel this `ss` can be immediately closed because this might be a hack to update `newHiveConf`
created_at: 2025-02-28T14:24:10Z
author_association: CONTRIBUTOR
diff_hunk: @@ -133,6 +135,13 @@ private void applyAuthorizationConfigPolicy(HiveConf newHiveConf) throws HiveExc
     ss.setIsHiveServerQuery(true);
     SessionState.start(ss);
     ss.applyAuthorizationPolicy();
path: service/src/java/org/apache/hive/service/cli/CLIService.java
user: login: Indhumathi27
body: you are right. we could close this sessionstate after applyAuthorizationPolicy
created_at: 2025-03-03T16:05:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -204,6 +205,12 @@ public void run() {
         } else {
           LOG.error("Problem connecting to the metastore when initializing the view registry", e);
         }
+      } finally {
+          try {
+              ss.close();
path: ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
user: login: deniskuzZ
body: could we make SessionState implement AutoCloseable and use try-with-resources?
created_at: 2025-03-04T10:49:16Z
author_association: MEMBER
diff_hunk: @@ -126,13 +126,14 @@ public synchronized void init(HiveConf hiveConf) {
   }
 
   private void applyAuthorizationConfigPolicy(HiveConf newHiveConf) throws HiveException,
-      MetaException {
+          MetaException, IOException {
     // authorization setup using SessionState should be revisited eventually, as
     // authorization and authentication are not session specific settings
     SessionState ss = new SessionState(newHiveConf);
     ss.setIsHiveServerQuery(true);
     SessionState.start(ss);
     ss.applyAuthorizationPolicy();
+    ss.close();
path: service/src/java/org/apache/hive/service/cli/CLIService.java
user: login: deniskuzZ
body: same
````
try (SessionState ss = new SessionState(newHiveConf)) {
...
}
````
created_at: 2025-03-04T10:50:04Z
author_association: MEMBER
diff_hunk: @@ -133,6 +135,13 @@ private void applyAuthorizationConfigPolicy(HiveConf newHiveConf) throws HiveExc
     ss.setIsHiveServerQuery(true);
     SessionState.start(ss);
     ss.applyAuthorizationPolicy();
path: service/src/java/org/apache/hive/service/cli/CLIService.java
user: login: Indhumathi27
body: I tried closing session in this case. Though the session is actually getting used in some cases like PrivilegeSynchonizer. Hence it has to be closed when the HS2 stop is called
created_at: 2025-03-04T11:39:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -204,6 +205,12 @@ public void run() {
         } else {
           LOG.error("Problem connecting to the metastore when initializing the view registry", e);
         }
+      } finally {
+          try {
+              ss.close();
path: ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
user: login: Indhumathi27
body: done
created_at: 2025-03-04T13:07:12Z
author_association: CONTRIBUTOR
diff_hunk: @@ -126,13 +126,14 @@ public synchronized void init(HiveConf hiveConf) {
   }
 
   private void applyAuthorizationConfigPolicy(HiveConf newHiveConf) throws HiveException,
-      MetaException {
+          MetaException, IOException {
     // authorization setup using SessionState should be revisited eventually, as
     // authorization and authentication are not session specific settings
     SessionState ss = new SessionState(newHiveConf);
     ss.setIsHiveServerQuery(true);
     SessionState.start(ss);
     ss.applyAuthorizationPolicy();
+    ss.close();
path: service/src/java/org/apache/hive/service/cli/CLIService.java
user: login: Indhumathi27
body: Some test scenarios like PrivilegeSynchronizer, Tez session ACL setup fails if the session is closed just after applyAuthorizationPolicy. Hence added a shutdownhook to close this session
created_at: 2025-03-04T13:08:52Z
author_association: CONTRIBUTOR
diff_hunk: @@ -126,13 +126,14 @@ public synchronized void init(HiveConf hiveConf) {
   }
 
   private void applyAuthorizationConfigPolicy(HiveConf newHiveConf) throws HiveException,
-      MetaException {
+          MetaException, IOException {
     // authorization setup using SessionState should be revisited eventually, as
     // authorization and authentication are not session specific settings
     SessionState ss = new SessionState(newHiveConf);
     ss.setIsHiveServerQuery(true);
     SessionState.start(ss);
     ss.applyAuthorizationPolicy();
+    ss.close();
path: service/src/java/org/apache/hive/service/cli/CLIService.java
user: login: deniskuzZ
body: any q test failures? maybe it is better to fix the tests and start/close session there?
created_at: 2025-03-05T05:51:14Z
author_association: MEMBER
diff_hunk: @@ -126,13 +126,14 @@ public synchronized void init(HiveConf hiveConf) {
   }
 
   private void applyAuthorizationConfigPolicy(HiveConf newHiveConf) throws HiveException,
-      MetaException {
+          MetaException, IOException {
     // authorization setup using SessionState should be revisited eventually, as
     // authorization and authentication are not session specific settings
     SessionState ss = new SessionState(newHiveConf);
     ss.setIsHiveServerQuery(true);
     SessionState.start(ss);
     ss.applyAuthorizationPolicy();
+    ss.close();
path: service/src/java/org/apache/hive/service/cli/CLIService.java
user: login: Indhumathi27
body: @deniskuzZ There are few testcase failures. For eg. org.apache.hive.jdbc.AbstractJdbcTriggersTest#miniHS2.start throws NPE if the session is closed. we are doing org.apache.hadoop.hive.ql.exec.tez.TezSessionState#setupSessionAcls for the active session state. 
Hence, in these cases, Sessionstate can't be closed prior.
created_at: 2025-03-11T16:29:39Z
author_association: CONTRIBUTOR
diff_hunk: @@ -126,13 +126,14 @@ public synchronized void init(HiveConf hiveConf) {
   }
 
   private void applyAuthorizationConfigPolicy(HiveConf newHiveConf) throws HiveException,
-      MetaException {
+          MetaException, IOException {
     // authorization setup using SessionState should be revisited eventually, as
     // authorization and authentication are not session specific settings
     SessionState ss = new SessionState(newHiveConf);
     ss.setIsHiveServerQuery(true);
     SessionState.start(ss);
     ss.applyAuthorizationPolicy();
+    ss.close();
path: service/src/java/org/apache/hive/service/cli/CLIService.java
user: login: deniskuzZ
body: ````  
/**
   * set current session to existing session object if a thread is running
   * multiple sessions - it must call this method with the new session object
   * when switching from one session to another.
   */
  public static SessionState start(SessionState startSs) {
    start(startSs, false, null);
    return startSs;
  }
````
It seems that In multiple places we just attach a new session object to the thread and don't close it. Is that true?
created_at: 2025-03-11T17:41:13Z
author_association: MEMBER
diff_hunk: @@ -126,13 +126,14 @@ public synchronized void init(HiveConf hiveConf) {
   }
 
   private void applyAuthorizationConfigPolicy(HiveConf newHiveConf) throws HiveException,
-      MetaException {
+          MetaException, IOException {
     // authorization setup using SessionState should be revisited eventually, as
     // authorization and authentication are not session specific settings
     SessionState ss = new SessionState(newHiveConf);
     ss.setIsHiveServerQuery(true);
     SessionState.start(ss);
     ss.applyAuthorizationPolicy();
+    ss.close();
path: service/src/java/org/apache/hive/service/cli/CLIService.java
user: login: deniskuzZ
body: 1. DriverUtils#setUpSessionState (StatsUpdate#gatherStats, DriverUtils#runOnDriverInternal)
2. WorkloadManager#scheduleWork
3. KillTriggerActionHandler#applyAction
4. HiveMaterializedViewsRegistry.Loader#run
5. CLIService#applyAuthorizationConfigPolicy
created_at: 2025-03-11T17:58:53Z
author_association: MEMBER
diff_hunk: @@ -126,13 +126,14 @@ public synchronized void init(HiveConf hiveConf) {
   }
 
   private void applyAuthorizationConfigPolicy(HiveConf newHiveConf) throws HiveException,
-      MetaException {
+          MetaException, IOException {
     // authorization setup using SessionState should be revisited eventually, as
     // authorization and authentication are not session specific settings
     SessionState ss = new SessionState(newHiveConf);
     ss.setIsHiveServerQuery(true);
     SessionState.start(ss);
     ss.applyAuthorizationPolicy();
+    ss.close();
path: service/src/java/org/apache/hive/service/cli/CLIService.java
user: login: deniskuzZ
body: @kasakrisz, do you know why in `HiveMaterializedViewsRegistry.Loader#run` we create a new SessionState object but never close it? 
created_at: 2025-03-11T18:02:49Z
author_association: MEMBER
diff_hunk: @@ -126,13 +126,14 @@ public synchronized void init(HiveConf hiveConf) {
   }
 
   private void applyAuthorizationConfigPolicy(HiveConf newHiveConf) throws HiveException,
-      MetaException {
+          MetaException, IOException {
     // authorization setup using SessionState should be revisited eventually, as
     // authorization and authentication are not session specific settings
     SessionState ss = new SessionState(newHiveConf);
     ss.setIsHiveServerQuery(true);
     SessionState.start(ss);
     ss.applyAuthorizationPolicy();
+    ss.close();
path: service/src/java/org/apache/hive/service/cli/CLIService.java
user: login: kasakrisz
body: `HiveMaterializedViewsRegistry.Loader` is a background thread for loading and compiling materialized views enabled for automatic query rewrite. The thread is started at HS2 start to inititalize the registry and it is also scheduled for periodic runs to update changes.
Seems that the only reason `SessionState.start` is called in this background thread is that we reach this point of code before it is called on the main thread at HS2 start up: https://issues.apache.org/jira/browse/HIVE-19691
The reason why a new `SessionState` instance is created is that we don't need TEZ session since no execution is happening in this thread: https://issues.apache.org/jira/browse/HIVE-21085

I'm thinking about whether we have to create new `SessionState` object at every refresh or it is required only at HS2 startup?

Anyway, I'm not aware of any reason cleaning of the `SessionState` was not added so I think it is a bug.

created_at: 2025-03-12T11:28:51Z
author_association: CONTRIBUTOR
diff_hunk: @@ -108,19 +108,16 @@ public boolean run(CompactorContext context) throws IOException, HiveException,
           VirtualColumn.FILE_PATH.getName(), fileSizePredicate == null ? "" : "and " + fileSizePredicate, orderBy);
     }
 
-    SessionState sessionState = setupQueryCompactionSession(conf, ci, tblProperties);
     String compactionTarget = "table " + HiveUtils.unparseIdentifier(compactTableName) +
-        (partSpec != null ? ", partition " + HiveUtils.unparseIdentifier(partSpec) : "");
+            (partSpec != null ? ", partition " + HiveUtils.unparseIdentifier(partSpec) : "");
 
-    try {
+    try (SessionState sessionState = setupQueryCompactionSession(conf, ci, tblProperties)) {
path: iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergQueryCompactor.java
user: login: deniskuzZ
body: that looks valid to me, however, I am a bit confused as to why we had to reset compaction marker on a session object in finally??
created_at: 2025-03-13T12:20:50Z
author_association: MEMBER
diff_hunk: @@ -486,15 +486,16 @@ private void scheduleWork(WmThreadSyncWork context) {
             LOG.info("Invoking KillQuery for " + queryId + ": " + reason);
             try {
               UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-              SessionState ss = new SessionState(new HiveConf(), ugi.getShortUserName());
-              ss.setIsHiveServerQuery(true);
-              SessionState.start(ss);
-              kq.killQuery(queryId, reason, toKill.getConf());
-              addKillQueryResult(toKill, true);
-              killCtx.killSessionFuture.set(true);
-              wmEvent.endEvent(toKill);
-              LOG.debug("Killed " + queryId);
-              return;
+                try (SessionState ss = new SessionState(new HiveConf(), ugi.getShortUserName())) {
path: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManager.java
user: login: deniskuzZ
body: looks legit to me. @abstractdog could you please confirm that's ok to close session object here?
created_at: 2025-03-13T12:23:43Z
author_association: MEMBER
diff_hunk: @@ -91,8 +91,9 @@ public void gatherStats(CompactionInfo ci, HiveConf hiveConf,
             if (compactionQueueName != null && compactionQueueName.length() > 0) {
                 conf.set(TezConfiguration.TEZ_QUEUE_NAME, compactionQueueName);
             }
-            SessionState sessionState = DriverUtils.setUpSessionState(conf, userName, true);
-            DriverUtils.runOnDriver(conf, sessionState, sb.toString());
+            try (SessionState sessionState = DriverUtils.setUpSessionState(conf, userName, true)) {
path: ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/StatsUpdater.java
user: login: deniskuzZ
body: `DriverUtils.setUpSessionState` supports session object reuse, but now we always create. a new one?
created_at: 2025-03-13T12:30:15Z
author_association: MEMBER
diff_hunk: @@ -91,8 +91,9 @@ public void gatherStats(CompactionInfo ci, HiveConf hiveConf,
             if (compactionQueueName != null && compactionQueueName.length() > 0) {
                 conf.set(TezConfiguration.TEZ_QUEUE_NAME, compactionQueueName);
             }
-            SessionState sessionState = DriverUtils.setUpSessionState(conf, userName, true);
-            DriverUtils.runOnDriver(conf, sessionState, sb.toString());
+            try (SessionState sessionState = DriverUtils.setUpSessionState(conf, userName, true)) {
path: ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/StatsUpdater.java
user: login: Indhumathi27
body: you are right. Actually in all this three cases, StatsUpdater, QueryCompactor and IcebergQueryCompactor, we are using DriverUtils.setUpSessionState. So, the session state can't be closed in run method. Can we add a shutdown hook to close session state on HS2 stop ? @deniskuzZ 
created_at: 2025-03-13T12:36:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -153,7 +153,7 @@ void runCompactionQueries(HiveConf conf, String tmpTableName, CompactionInfo com
         LOG.error(ExceptionUtils.getStackTrace(e));
       } finally {
         //restore sessionState
-        sessionState.setCompaction(false);
+        sessionState.close();
path: ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java
user: login: deniskuzZ
body: why not move it under try-with-resources?
created_at: 2025-03-13T12:42:42Z
author_association: MEMBER
diff_hunk: @@ -153,7 +153,7 @@ void runCompactionQueries(HiveConf conf, String tmpTableName, CompactionInfo com
         LOG.error(ExceptionUtils.getStackTrace(e));
       } finally {
         //restore sessionState
-        sessionState.setCompaction(false);
+        sessionState.close();
path: ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java
user: login: Indhumathi27
body: SessionState is reused in finally block as well
created_at: 2025-03-13T12:44:45Z
author_association: CONTRIBUTOR
diff_hunk: @@ -91,8 +91,9 @@ public void gatherStats(CompactionInfo ci, HiveConf hiveConf,
             if (compactionQueueName != null && compactionQueueName.length() > 0) {
                 conf.set(TezConfiguration.TEZ_QUEUE_NAME, compactionQueueName);
             }
-            SessionState sessionState = DriverUtils.setUpSessionState(conf, userName, true);
-            DriverUtils.runOnDriver(conf, sessionState, sb.toString());
+            try (SessionState sessionState = DriverUtils.setUpSessionState(conf, userName, true)) {
path: ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/StatsUpdater.java
user: login: deniskuzZ
body: I think so
created_at: 2025-03-13T12:47:24Z
author_association: MEMBER
diff_hunk: @@ -108,19 +108,16 @@ public boolean run(CompactorContext context) throws IOException, HiveException,
           VirtualColumn.FILE_PATH.getName(), fileSizePredicate == null ? "" : "and " + fileSizePredicate, orderBy);
     }
 
-    SessionState sessionState = setupQueryCompactionSession(conf, ci, tblProperties);
     String compactionTarget = "table " + HiveUtils.unparseIdentifier(compactTableName) +
-        (partSpec != null ? ", partition " + HiveUtils.unparseIdentifier(partSpec) : "");
+            (partSpec != null ? ", partition " + HiveUtils.unparseIdentifier(partSpec) : "");
 
-    try {
+    try (SessionState sessionState = setupQueryCompactionSession(conf, ci, tblProperties)) {
path: iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergQueryCompactor.java
user: login: deniskuzZ
body: this -> https://github.com/apache/hive/pull/5649/files#r1993430862
created_at: 2025-03-13T12:48:25Z
author_association: MEMBER
diff_hunk: @@ -153,7 +153,7 @@ void runCompactionQueries(HiveConf conf, String tmpTableName, CompactionInfo com
         LOG.error(ExceptionUtils.getStackTrace(e));
       } finally {
         //restore sessionState
-        sessionState.setCompaction(false);
+        sessionState.close();
path: ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java
user: login: deniskuzZ
body: yes, we shouldn't close
created_at: 2025-03-13T12:49:36Z
author_association: MEMBER
diff_hunk: @@ -133,6 +134,15 @@ private void applyAuthorizationConfigPolicy(HiveConf newHiveConf) throws HiveExc
     ss.setIsHiveServerQuery(true);
     SessionState.start(ss);
     ss.applyAuthorizationPolicy();
+
+    // Add shutdown hook to close the session.
+    ShutdownHookManager.addShutdownHook(() -> {
path: service/src/java/org/apache/hive/service/cli/CLIService.java
user: login: abstractdog
body: does it make sense to introduce something like:
```
ss.closeOnExit()
```
which will be reusable on the SessionState object level
created_at: 2025-03-13T13:47:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -126,13 +126,14 @@ public synchronized void init(HiveConf hiveConf) {
   }
 
   private void applyAuthorizationConfigPolicy(HiveConf newHiveConf) throws HiveException,
-      MetaException {
+          MetaException, IOException {
     // authorization setup using SessionState should be revisited eventually, as
     // authorization and authentication are not session specific settings
     SessionState ss = new SessionState(newHiveConf);
     ss.setIsHiveServerQuery(true);
     SessionState.start(ss);
     ss.applyAuthorizationPolicy();
+    ss.close();
path: service/src/java/org/apache/hive/service/cli/CLIService.java
user: login: abstractdog
body: I agree that attaching a new session state to threadlocal without closing the previous one could be a problem, but creating + closing that every time properly might also be a problem from performance point of view (just taking a look at how much code resides in SesssionState.close), so wherever reusing a SessionState is possible, let's reuse, I guess having e.g. HiveMaterializedViewsRegistry as the outer scope we can store the SessionState as field reference and close it when HS2 is shut down
created_at: 2025-03-13T14:06:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -486,15 +486,16 @@ private void scheduleWork(WmThreadSyncWork context) {
             LOG.info("Invoking KillQuery for " + queryId + ": " + reason);
             try {
               UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-              SessionState ss = new SessionState(new HiveConf(), ugi.getShortUserName());
-              ss.setIsHiveServerQuery(true);
-              SessionState.start(ss);
-              kq.killQuery(queryId, reason, toKill.getConf());
-              addKillQueryResult(toKill, true);
-              killCtx.killSessionFuture.set(true);
-              wmEvent.endEvent(toKill);
-              LOG.debug("Killed " + queryId);
-              return;
+                try (SessionState ss = new SessionState(new HiveConf(), ugi.getShortUserName())) {
path: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManager.java
user: login: abstractdog
body: here, as I referred to in the case of the MV registry, what about trying a SessionState field in the WorkloadManager, so instead of:
```
              UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
              SessionState ss = new SessionState(new HiveConf(), ugi.getShortUserName());
```
we can initialize the SessionState when the WorkloadManager is created, because neither the object returned by "new HiveConf()", nor the "UserGroupInformation.getCurrentUser()" is supposed to change over time in HS2, it's safe to reuse the same SessionState
created_at: 2025-03-13T14:09:56Z
author_association: CONTRIBUTOR
diff_hunk: @@ -121,6 +121,7 @@ public boolean run(CompactorContext context) throws IOException, HiveException,
       throw e;
     } finally {
       sessionState.setCompaction(false);
+      sessionState.closeOnExit();
path: iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/compaction/IcebergQueryCompactor.java
user: login: deniskuzZ
body: shouldn't we set the hook when we create a session object?  i don't think that is the right place, since we reuse parent session here
created_at: 2025-03-13T18:53:12Z
author_association: MEMBER
diff_hunk: @@ -38,28 +40,27 @@ public class KillTriggerActionHandler implements TriggerActionHandler<TezSession
   @Override
   public void applyAction(final Map<TezSessionState, Trigger> queriesViolated) {
     for (Map.Entry<TezSessionState, Trigger> entry : queriesViolated.entrySet()) {
-      switch (entry.getValue().getAction().getType()) {
-      case KILL_QUERY:
-        TezSessionState sessionState = entry.getKey();
-        String queryId = sessionState.getWmContext().getQueryId();
-        try {
-          UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-          SessionState ss = new SessionState(new HiveConf(), ugi.getShortUserName());
-          ss.setIsHiveServerQuery(true);
-          SessionState.start(ss);
-          KillQuery killQuery = sessionState.getKillQuery();
-          // if kill query is null then session might have been released to pool or closed already
-          if (killQuery != null) {
-            sessionState.getKillQuery().killQuery(queryId, entry.getValue().getViolationMsg(),
-                      sessionState.getConf());
-          }
-        } catch (HiveException|IOException e) {
-          LOG.warn("Unable to kill query {} for trigger violation", queryId);
+        if (Objects.requireNonNull(entry.getValue().getAction().getType()) == Action.Type.KILL_QUERY) {
path: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillTriggerActionHandler.java
user: login: deniskuzZ
body: why do you need `Objects.requireNonNull`?

````
if (Action.Type.KILL_QUERY == entry.getValue().getAction().getType()) {
..
}
````
created_at: 2025-03-13T18:55:30Z
author_association: MEMBER
diff_hunk: @@ -38,28 +40,27 @@ public class KillTriggerActionHandler implements TriggerActionHandler<TezSession
   @Override
   public void applyAction(final Map<TezSessionState, Trigger> queriesViolated) {
     for (Map.Entry<TezSessionState, Trigger> entry : queriesViolated.entrySet()) {
-      switch (entry.getValue().getAction().getType()) {
-      case KILL_QUERY:
-        TezSessionState sessionState = entry.getKey();
-        String queryId = sessionState.getWmContext().getQueryId();
-        try {
-          UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-          SessionState ss = new SessionState(new HiveConf(), ugi.getShortUserName());
-          ss.setIsHiveServerQuery(true);
-          SessionState.start(ss);
-          KillQuery killQuery = sessionState.getKillQuery();
-          // if kill query is null then session might have been released to pool or closed already
-          if (killQuery != null) {
-            sessionState.getKillQuery().killQuery(queryId, entry.getValue().getViolationMsg(),
-                      sessionState.getConf());
-          }
-        } catch (HiveException|IOException e) {
-          LOG.warn("Unable to kill query {} for trigger violation", queryId);
+        if (Objects.requireNonNull(entry.getValue().getAction().getType()) == Action.Type.KILL_QUERY) {
+            TezSessionState sessionState = entry.getKey();
+            String queryId = sessionState.getWmContext().getQueryId();
+            try {
+                UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+                try (SessionState ss = new SessionState(new HiveConf(), ugi.getShortUserName())) {
path: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillTriggerActionHandler.java
user: login: deniskuzZ
body: similar to other comments, do we need to create and start new session per tez session entry?
````
DriverUtils.setUpSessionState(new HiveConf(), ugi.getShortUserName(), true)
````
created_at: 2025-03-13T18:58:32Z
author_association: MEMBER
diff_hunk: @@ -331,6 +334,7 @@ public void stop() throws Exception {
       stopTriggerValidator();
     }
     INSTANCE = null;
+    sessionState.close();
path: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManager.java
user: login: deniskuzZ
body: null check is missing
created_at: 2025-03-13T18:59:27Z
author_association: MEMBER
diff_hunk: @@ -154,6 +154,7 @@ void runCompactionQueries(HiveConf conf, String tmpTableName, CompactionInfo com
       } finally {
         //restore sessionState
         sessionState.setCompaction(false);
+        sessionState.closeOnExit();
path: ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java
user: login: deniskuzZ
body: not required here
created_at: 2025-03-13T19:00:30Z
author_association: MEMBER
diff_hunk: @@ -93,6 +93,7 @@ public void gatherStats(CompactionInfo ci, HiveConf hiveConf,
             }
             SessionState sessionState = DriverUtils.setUpSessionState(conf, userName, true);
             DriverUtils.runOnDriver(conf, sessionState, sb.toString());
+            sessionState.closeOnExit();
path: ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/StatsUpdater.java
user: login: deniskuzZ
body: i think it should. be done inside of `DriverUtils.setUpSessionState`
created_at: 2025-03-13T19:02:42Z
author_association: MEMBER
diff_hunk: @@ -167,12 +168,11 @@ private Loader(Hive db) {
 
     @Override
     public void run() {
-      SessionState ss = new SessionState(db.getConf());
-      ss.setIsHiveServerQuery(true); // All is served from HS2, we do not need e.g. Tez sessions
-      SessionState.start(ss);
       PerfLogger perfLogger = SessionState.getPerfLogger();
-      perfLogger.perfLogBegin(CLASS_NAME, PerfLogger.MATERIALIZED_VIEWS_REGISTRY_REFRESH);
-      try {
+      try (SessionState ss = new SessionState(db.getConf())) {
path: ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
user: login: deniskuzZ
body: should we use `DriverUtils.setUpSessionState(db.getConf(), null, true)`
created_at: 2025-03-13T19:05:03Z
author_association: MEMBER
diff_hunk: @@ -203,12 +204,14 @@ public static WorkloadManager create(String yarnQueue, HiveConf conf, WMFullReso
 
   @VisibleForTesting
   WorkloadManager(LlapPluginEndpointClientImpl amComm, String yarnQueue, HiveConf conf,
-      QueryAllocationManager qam, WMFullResourcePlan plan) throws ExecutionException, InterruptedException {
+      QueryAllocationManager qam, WMFullResourcePlan plan) throws ExecutionException, InterruptedException, IOException {
     this.yarnQueue = yarnQueue;
     this.conf = conf;
     this.totalQueryParallelism = determineQueryParallelism(plan);
     this.allocationManager = qam;
     this.allocationManager.setClusterChangedCallback(() -> notifyOfClusterStateChange());
+    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+    this.sessionState = new SessionState(new HiveConf(), ugi.getShortUserName());
path: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManager.java
user: login: deniskuzZ
body: no need for eager initialization, use lazy when really needed
````
DriverUtils.setUpSessionState(new HiveConf(), ugi.getShortUserName(), true)
````
created_at: 2025-03-13T19:12:58Z
author_association: MEMBER
diff_hunk: @@ -38,28 +37,23 @@ public class KillTriggerActionHandler implements TriggerActionHandler<TezSession
   @Override
   public void applyAction(final Map<TezSessionState, Trigger> queriesViolated) {
     for (Map.Entry<TezSessionState, Trigger> entry : queriesViolated.entrySet()) {
-      switch (entry.getValue().getAction().getType()) {
-      case KILL_QUERY:
-        TezSessionState sessionState = entry.getKey();
-        String queryId = sessionState.getWmContext().getQueryId();
-        try {
-          UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-          SessionState ss = new SessionState(new HiveConf(), ugi.getShortUserName());
-          ss.setIsHiveServerQuery(true);
-          SessionState.start(ss);
-          KillQuery killQuery = sessionState.getKillQuery();
-          // if kill query is null then session might have been released to pool or closed already
-          if (killQuery != null) {
-            sessionState.getKillQuery().killQuery(queryId, entry.getValue().getViolationMsg(),
-                      sessionState.getConf());
-          }
-        } catch (HiveException|IOException e) {
-          LOG.warn("Unable to kill query {} for trigger violation", queryId);
+        if (entry.getValue().getAction().getType() == Action.Type.KILL_QUERY) {
+            TezSessionState sessionState = entry.getKey();
+            String queryId = sessionState.getWmContext().getQueryId();
+            try {
+                DriverUtils.setUpSessionState(sessionState.getConf(),true);
path: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillTriggerActionHandler.java
user: login: deniskuzZ
body: missing space between args
created_at: 2025-03-17T15:46:03Z
author_association: MEMBER
diff_hunk: @@ -485,17 +487,15 @@ private void scheduleWork(WmThreadSyncWork context) {
             WmEvent wmEvent = new WmEvent(WmEvent.EventType.KILL);
             LOG.info("Invoking KillQuery for " + queryId + ": " + reason);
             try {
-              UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-              SessionState ss = new SessionState(new HiveConf(), ugi.getShortUserName());
-              ss.setIsHiveServerQuery(true);
-              SessionState.start(ss);
+              sessionState.setIsHiveServerQuery(true);
path: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManager.java
user: login: deniskuzZ
body: why not do `DriverUtils.setUpSessionState(conf, true)` here?
created_at: 2025-03-17T15:47:09Z
author_association: MEMBER
diff_hunk: @@ -104,8 +110,8 @@ public static SessionState setUpSessionState(HiveConf conf, String user, boolean
         SessionState.start(sessionState);
       }
       SessionState.setCurrentSessionState(sessionState);
-    }
-    else {
+      sessionState.closeOnExit();
path: ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java
user: login: deniskuzZ
body: @Indhumathi27, what if we make it generic and add shutdown hook only for started sessions, i.e. register hook in `SessionState.start`?
created_at: 2025-03-17T15:52:03Z
author_association: MEMBER
diff_hunk: @@ -92,6 +91,11 @@ private static void runOnDriverInternal(String query, HiveConf conf, SessionStat
     }
   }
 
+  public static SessionState setUpSessionState(HiveConf conf, boolean doStart) throws IOException {
path: ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java
user: login: deniskuzZ
body: @Indhumathi27, i think it's better to pass user explicitly and use 'null' when not provided, default doStart=true
````
- DriverUtils.setUpSessionState(sessionState.getConf(), ugi.getShortUserName)
- DriverUtils.setUpSessionState(sessionState.getConf())
````
created_at: 2025-03-18T08:37:39Z
author_association: MEMBER
diff_hunk: @@ -92,6 +91,11 @@ private static void runOnDriverInternal(String query, HiveConf conf, SessionStat
     }
   }
 
+  public static SessionState setUpSessionState(HiveConf conf, boolean doStart) throws IOException {
+    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+    return setUpSessionState(conf, ugi.getUserName(), doStart);
path: ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java
user: login: deniskuzZ
body: should it be `ugi.getShortUserName`?
created_at: 2025-03-18T08:42:29Z
author_association: MEMBER
diff_hunk: @@ -167,12 +169,10 @@ private Loader(Hive db) {
 
     @Override
     public void run() {
-      SessionState ss = new SessionState(db.getConf());
-      ss.setIsHiveServerQuery(true); // All is served from HS2, we do not need e.g. Tez sessions
-      SessionState.start(ss);
       PerfLogger perfLogger = SessionState.getPerfLogger();
-      perfLogger.perfLogBegin(CLASS_NAME, PerfLogger.MATERIALIZED_VIEWS_REGISTRY_REFRESH);
       try {
+        DriverUtils.setUpSessionState(db.getConf(), true);
path: ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
user: login: deniskuzZ
body: should we pass `null` user here?
created_at: 2025-03-18T08:50:23Z
author_association: MEMBER
diff_hunk: @@ -126,12 +127,10 @@ public synchronized void init(HiveConf hiveConf) {
   }
 
   private void applyAuthorizationConfigPolicy(HiveConf newHiveConf) throws HiveException,
-      MetaException {
+      MetaException, IOException {
     // authorization setup using SessionState should be revisited eventually, as
     // authorization and authentication are not session specific settings
-    SessionState ss = new SessionState(newHiveConf);
-    ss.setIsHiveServerQuery(true);
-    SessionState.start(ss);
+    SessionState ss = DriverUtils.setUpSessionState(newHiveConf, true);
path: service/src/java/org/apache/hive/service/cli/CLIService.java
user: login: deniskuzZ
body: should we pass `null` user here?
created_at: 2025-03-18T08:51:11Z
author_association: MEMBER
diff_hunk: @@ -168,6 +168,7 @@ public class WorkloadManager extends TezSessionPoolSession.AbstractTriggerValida
   private final Thread delayedMoveThread;
   private final int delayedMoveTimeOutSec;
   private final int delayedMoveValidationIntervalSec;
+  private final SessionState sessionState;
path: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManager.java
user: login: deniskuzZ
body: why should it be an instance property?
created_at: 2025-03-18T09:19:44Z
author_association: MEMBER
diff_hunk: @@ -168,6 +168,7 @@ public class WorkloadManager extends TezSessionPoolSession.AbstractTriggerValida
   private final Thread delayedMoveThread;
   private final int delayedMoveTimeOutSec;
   private final int delayedMoveValidationIntervalSec;
+  private final SessionState sessionState;
path: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManager.java
user: login: Indhumathi27
body: post handling comments, forget to remove this. will work on it
created_at: 2025-03-18T09:21:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -168,6 +168,7 @@ public class WorkloadManager extends TezSessionPoolSession.AbstractTriggerValida
   private final Thread delayedMoveThread;
   private final int delayedMoveTimeOutSec;
   private final int delayedMoveValidationIntervalSec;
+  private final SessionState sessionState;
path: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManager.java
user: login: deniskuzZ
body: it was good until the last force-push 
created_at: 2025-03-18T09:23:02Z
author_association: MEMBER
diff_hunk: @@ -168,6 +168,7 @@ public class WorkloadManager extends TezSessionPoolSession.AbstractTriggerValida
   private final Thread delayedMoveThread;
   private final int delayedMoveTimeOutSec;
   private final int delayedMoveValidationIntervalSec;
+  private final SessionState sessionState;
path: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManager.java
user: login: deniskuzZ
body: please check https://github.com/apache/hive/pull/5649/files#r2000475143 as well
created_at: 2025-03-18T09:23:49Z
author_association: MEMBER
diff_hunk: @@ -203,12 +203,13 @@ public static WorkloadManager create(String yarnQueue, HiveConf conf, WMFullReso
 
   @VisibleForTesting
   WorkloadManager(LlapPluginEndpointClientImpl amComm, String yarnQueue, HiveConf conf,
-      QueryAllocationManager qam, WMFullResourcePlan plan) throws ExecutionException, InterruptedException {
+      QueryAllocationManager qam, WMFullResourcePlan plan) throws ExecutionException, InterruptedException, IOException {
     this.yarnQueue = yarnQueue;
     this.conf = conf;
     this.totalQueryParallelism = determineQueryParallelism(plan);
     this.allocationManager = qam;
     this.allocationManager.setClusterChangedCallback(() -> notifyOfClusterStateChange());
+    DriverUtils.setUpSessionState(conf, true);
path: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/WorkloadManager.java
user: login: deniskuzZ
body: why did it migrate to the constructor?
created_at: 2025-03-18T09:32:23Z
author_association: MEMBER
diff_hunk: @@ -92,6 +91,11 @@ private static void runOnDriverInternal(String query, HiveConf conf, SessionStat
     }
   }
 
+  public static SessionState setUpSessionState(HiveConf conf, boolean doStart) throws IOException {
path: ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java
user: login: Indhumathi27
body: doStart is false in one case, StatsUpdaterThread.run. Hence didnt change that method signature.
Added,
- DriverUtils.setUpSessionState(sessionState.getConf(), ugi.getShortUserName)
- DriverUtils.setUpSessionState(sessionState.getConf())
created_at: 2025-03-18T10:21:29Z
author_association: CONTRIBUTOR
diff_hunk: @@ -92,6 +90,14 @@ private static void runOnDriverInternal(String query, HiveConf conf, SessionStat
     }
   }
 
+  public static SessionState setUpSessionState(HiveConf conf) {
+    return setUpSessionState(conf, null, true);
path: ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java
user: login: deniskuzZ
body: ```return setUpSessionState(conf, null)```
created_at: 2025-03-18T12:48:28Z
author_association: MEMBER
diff_hunk: @@ -104,8 +110,7 @@ public static SessionState setUpSessionState(HiveConf conf, String user, boolean
         SessionState.start(sessionState);
       }
       SessionState.setCurrentSessionState(sessionState);
path: ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java
user: login: deniskuzZ
body: Do you know if that is required? i think we can drop it since it's set inside `start`
created_at: 2025-03-18T13:05:35Z
author_association: MEMBER
diff_hunk: @@ -1969,6 +1974,18 @@ public void close() throws IOException {
     dynamicVars.clear();
   }
 
+  public void addSessionStateShutdownHook() {
path: ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
user: login: deniskuzZ
body: ````
private void addShutdownHook()
````
created_at: 2025-03-18T16:00:36Z
author_association: MEMBER
diff_hunk: @@ -341,6 +345,29 @@ public enum AuthorizationMode{V1, V2};
    */
   private boolean compaction = false;
 
+  // A thread-safe set to hold all active session states
+  private static final Set<SessionState> sessionStates = Collections.newSetFromMap(new ConcurrentHashMap<>());
+
+  // Thread pool for concurrent cleanup
+  private static final ExecutorService cleanupExecutor = Executors.newCachedThreadPool();
+
+  // Static block to add a single shutdown hook for all session states
+  static {
+    ShutdownHookManager.addShutdownHook(new Thread(() -> {
path: ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
user: login: deniskuzZ
body: can't we pass Runnable? why are you creating a new Thread?
````
ShutdownHookManager.addShutdownHook(SessionState::cleanUpAllSessions);
````
created_at: 2025-03-27T19:02:01Z
author_association: MEMBER
diff_hunk: @@ -341,6 +345,29 @@ public enum AuthorizationMode{V1, V2};
    */
   private boolean compaction = false;
 
+  // A thread-safe set to hold all active session states
+  private static final Set<SessionState> sessionStates = Collections.newSetFromMap(new ConcurrentHashMap<>());
path: ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
user: login: deniskuzZ
body: ConcurrentHashMap.newKeySet()
created_at: 2025-03-27T19:06:41Z
author_association: MEMBER
diff_hunk: @@ -341,6 +345,29 @@ public enum AuthorizationMode{V1, V2};
    */
   private boolean compaction = false;
 
+  // A thread-safe set to hold all active session states
+  private static final Set<SessionState> sessionStates = Collections.newSetFromMap(new ConcurrentHashMap<>());
+
+  // Thread pool for concurrent cleanup
+  private static final ExecutorService cleanupExecutor = Executors.newCachedThreadPool();
+
+  // Static block to add a single shutdown hook for all session states
+  static {
+    ShutdownHookManager.addShutdownHook(new Thread(() -> {
+      try {
+        cleanUpAllSessions();
path: ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
user: login: deniskuzZ
body: why don't we create and destroy the threadpool inside of `cleanUpAllSessions`?
created_at: 2025-03-27T19:08:30Z
author_association: MEMBER
diff_hunk: @@ -1969,6 +2001,24 @@ public void close() throws IOException {
     dynamicVars.clear();
   }
 
+  private static void deregister(SessionState ss) {
+    // remove from the sessionStates list to avoid cleaning the closed session state again
+    sessionStates.remove(ss);
+  }
+
+  private static void cleanUpAllSessions() {
+    for (SessionState sessionState : sessionStates) {
+      cleanupExecutor.submit(() -> {
path: ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
user: login: deniskuzZ
body: where do you wait for completion?
````
private static void cleanUpAllSessions() {
  ExecutorService cleanupExecutor = Executors.newCachedThreadPool();
  try {
    List<CompletableFuture<Void>> asyncTasks = new ArrayList<>();
    for (SessionState sessionState : sessionStates) {
      Runnable task = () -> {
        try {
          LOG.info("Closing session state: {}", sessionState.getSessionId());
          sessionState.close();
        } catch (IOException e) {
          LOG.error("Problem closing session state", e);
        }
      };
      CompletableFuture<Void> asyncTask = CompletableFuture.runAsync(
          task, cleanupExecutor)
        .exceptionally(e -> {
          LOG.error("Error closing session state", e);
          return null;
        });
      asyncTasks.add(asyncTask);
    }
    CompletableFuture<Void> allTasks = CompletableFuture.allOf(asyncTasks.toArray(new CompletableFuture[0]));
    try {
      allTasks.get(60, TimeUnit.SECONDS);
    } catch (Exception e) {
      LOG.error("Failed to close all session states", e);
    }
  } finally {
    cleanupExecutor.shutdownNow();
  }
}
````
created_at: 2025-03-27T19:09:55Z
author_association: MEMBER
diff_hunk: @@ -1969,6 +2001,24 @@ public void close() throws IOException {
     dynamicVars.clear();
   }
 
+  private static void deregister(SessionState ss) {
path: ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
user: login: deniskuzZ
body: why do you need this 1-liner method, just call `sessionStates.remove(this);`
created_at: 2025-03-27T19:49:24Z
author_association: MEMBER
diff_hunk: @@ -38,28 +38,24 @@ public class KillTriggerActionHandler implements TriggerActionHandler<TezSession
   @Override
   public void applyAction(final Map<TezSessionState, Trigger> queriesViolated) {
     for (Map.Entry<TezSessionState, Trigger> entry : queriesViolated.entrySet()) {
-      switch (entry.getValue().getAction().getType()) {
-      case KILL_QUERY:
-        TezSessionState sessionState = entry.getKey();
-        String queryId = sessionState.getWmContext().getQueryId();
-        try {
-          UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-          SessionState ss = new SessionState(new HiveConf(), ugi.getShortUserName());
-          ss.setIsHiveServerQuery(true);
-          SessionState.start(ss);
-          KillQuery killQuery = sessionState.getKillQuery();
-          // if kill query is null then session might have been released to pool or closed already
-          if (killQuery != null) {
-            sessionState.getKillQuery().killQuery(queryId, entry.getValue().getViolationMsg(),
-                      sessionState.getConf());
-          }
-        } catch (HiveException|IOException e) {
-          LOG.warn("Unable to kill query {} for trigger violation", queryId);
+        if (entry.getValue().getAction().getType() == Action.Type.KILL_QUERY) {
+            TezSessionState sessionState = entry.getKey();
+            String queryId = sessionState.getWmContext().getQueryId();
+            try {
+                UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+                DriverUtils.setUpSessionState(sessionState.getConf(), ugi.getShortUserName());
path: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillTriggerActionHandler.java
user: login: deniskuzZ
body: Should we pass the `conf` to `KillTriggerActionHandler` constructor from `TezSessionPoolManager#initTriggers` and use it instead of `TezSessionState#getConf`?
created_at: 2025-03-27T20:43:22Z
author_association: MEMBER
diff_hunk: @@ -1969,6 +2001,24 @@ public void close() throws IOException {
     dynamicVars.clear();
   }
 
+  private static void deregister(SessionState ss) {
+    // remove from the sessionStates list to avoid cleaning the closed session state again
+    sessionStates.remove(ss);
+  }
+
+  private static void cleanUpAllSessions() {
+    for (SessionState sessionState : sessionStates) {
+      cleanupExecutor.submit(() -> {
path: ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
user: login: Indhumathi27
body: Thanks for suggesting @deniskuzZ . I have used this and added some changes on top of this code. please check
created_at: 2025-03-28T07:45:10Z
author_association: CONTRIBUTOR
diff_hunk: @@ -38,28 +38,24 @@ public class KillTriggerActionHandler implements TriggerActionHandler<TezSession
   @Override
   public void applyAction(final Map<TezSessionState, Trigger> queriesViolated) {
     for (Map.Entry<TezSessionState, Trigger> entry : queriesViolated.entrySet()) {
-      switch (entry.getValue().getAction().getType()) {
-      case KILL_QUERY:
-        TezSessionState sessionState = entry.getKey();
-        String queryId = sessionState.getWmContext().getQueryId();
-        try {
-          UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-          SessionState ss = new SessionState(new HiveConf(), ugi.getShortUserName());
-          ss.setIsHiveServerQuery(true);
-          SessionState.start(ss);
-          KillQuery killQuery = sessionState.getKillQuery();
-          // if kill query is null then session might have been released to pool or closed already
-          if (killQuery != null) {
-            sessionState.getKillQuery().killQuery(queryId, entry.getValue().getViolationMsg(),
-                      sessionState.getConf());
-          }
-        } catch (HiveException|IOException e) {
-          LOG.warn("Unable to kill query {} for trigger violation", queryId);
+        if (entry.getValue().getAction().getType() == Action.Type.KILL_QUERY) {
+            TezSessionState sessionState = entry.getKey();
+            String queryId = sessionState.getWmContext().getQueryId();
+            try {
+                UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+                DriverUtils.setUpSessionState(sessionState.getConf(), ugi.getShortUserName());
path: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillTriggerActionHandler.java
user: login: Indhumathi27
body: KillTriggerActionHandler - This one looks like, calls KillQuery respective to a TezSessionState. Shouldn't it use the conf respective to the same ?
created_at: 2025-03-28T07:57:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -38,28 +38,24 @@ public class KillTriggerActionHandler implements TriggerActionHandler<TezSession
   @Override
   public void applyAction(final Map<TezSessionState, Trigger> queriesViolated) {
     for (Map.Entry<TezSessionState, Trigger> entry : queriesViolated.entrySet()) {
-      switch (entry.getValue().getAction().getType()) {
-      case KILL_QUERY:
-        TezSessionState sessionState = entry.getKey();
-        String queryId = sessionState.getWmContext().getQueryId();
-        try {
-          UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-          SessionState ss = new SessionState(new HiveConf(), ugi.getShortUserName());
-          ss.setIsHiveServerQuery(true);
-          SessionState.start(ss);
-          KillQuery killQuery = sessionState.getKillQuery();
-          // if kill query is null then session might have been released to pool or closed already
-          if (killQuery != null) {
-            sessionState.getKillQuery().killQuery(queryId, entry.getValue().getViolationMsg(),
-                      sessionState.getConf());
-          }
-        } catch (HiveException|IOException e) {
-          LOG.warn("Unable to kill query {} for trigger violation", queryId);
+        if (entry.getValue().getAction().getType() == Action.Type.KILL_QUERY) {
+            TezSessionState sessionState = entry.getKey();
+            String queryId = sessionState.getWmContext().getQueryId();
+            try {
+                UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+                DriverUtils.setUpSessionState(sessionState.getConf(), ugi.getShortUserName());
path: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillTriggerActionHandler.java
user: login: deniskuzZ
body: Before, when we created a new session, we passed a `new HiveConf`, but now that might overwrite the current session conf. Note that in `WorkloadManager` you are using global conf and not the `TezSessionState`
IDK, maybe we should fallback to the original `new HiveConf`. It would be nice to understand in what Thread both this things are executed and if. SessionState exists when we enter there?
created_at: 2025-03-28T08:59:34Z
author_association: MEMBER
diff_hunk: @@ -38,28 +38,24 @@ public class KillTriggerActionHandler implements TriggerActionHandler<TezSession
   @Override
   public void applyAction(final Map<TezSessionState, Trigger> queriesViolated) {
     for (Map.Entry<TezSessionState, Trigger> entry : queriesViolated.entrySet()) {
-      switch (entry.getValue().getAction().getType()) {
-      case KILL_QUERY:
-        TezSessionState sessionState = entry.getKey();
-        String queryId = sessionState.getWmContext().getQueryId();
-        try {
-          UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-          SessionState ss = new SessionState(new HiveConf(), ugi.getShortUserName());
-          ss.setIsHiveServerQuery(true);
-          SessionState.start(ss);
-          KillQuery killQuery = sessionState.getKillQuery();
-          // if kill query is null then session might have been released to pool or closed already
-          if (killQuery != null) {
-            sessionState.getKillQuery().killQuery(queryId, entry.getValue().getViolationMsg(),
-                      sessionState.getConf());
-          }
-        } catch (HiveException|IOException e) {
-          LOG.warn("Unable to kill query {} for trigger violation", queryId);
+        if (entry.getValue().getAction().getType() == Action.Type.KILL_QUERY) {
+            TezSessionState sessionState = entry.getKey();
+            String queryId = sessionState.getWmContext().getQueryId();
+            try {
+                UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+                DriverUtils.setUpSessionState(sessionState.getConf(), ugi.getShortUserName());
path: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillTriggerActionHandler.java
user: login: Indhumathi27
body: Reverted back to the original way of using New HiveConf. This triggerValidatorRunnable is run as a separate thread(org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolSession.AbstractTriggerValidator#scheduledExecutorService) initiated in TezSessionPoolManager.
created_at: 2025-03-28T11:43:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -38,28 +38,24 @@ public class KillTriggerActionHandler implements TriggerActionHandler<TezSession
   @Override
   public void applyAction(final Map<TezSessionState, Trigger> queriesViolated) {
     for (Map.Entry<TezSessionState, Trigger> entry : queriesViolated.entrySet()) {
-      switch (entry.getValue().getAction().getType()) {
-      case KILL_QUERY:
-        TezSessionState sessionState = entry.getKey();
-        String queryId = sessionState.getWmContext().getQueryId();
-        try {
-          UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-          SessionState ss = new SessionState(new HiveConf(), ugi.getShortUserName());
-          ss.setIsHiveServerQuery(true);
-          SessionState.start(ss);
-          KillQuery killQuery = sessionState.getKillQuery();
-          // if kill query is null then session might have been released to pool or closed already
-          if (killQuery != null) {
-            sessionState.getKillQuery().killQuery(queryId, entry.getValue().getViolationMsg(),
-                      sessionState.getConf());
-          }
-        } catch (HiveException|IOException e) {
-          LOG.warn("Unable to kill query {} for trigger violation", queryId);
+        if (entry.getValue().getAction().getType() == Action.Type.KILL_QUERY) {
+            TezSessionState sessionState = entry.getKey();
+            String queryId = sessionState.getWmContext().getQueryId();
+            try {
+                UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+                DriverUtils.setUpSessionState(sessionState.getConf(), ugi.getShortUserName());
path: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillTriggerActionHandler.java
user: login: deniskuzZ
body: `WorkloadManager` also schedules work in a new thread (i.e. `new Thread(() -> runWmThread()`)



created_at: 2025-03-28T16:21:15Z
author_association: MEMBER
diff_hunk: @@ -104,8 +110,7 @@ public static SessionState setUpSessionState(HiveConf conf, String user, boolean
         SessionState.start(sessionState);
       }
       SessionState.setCurrentSessionState(sessionState);
-    }
-    else {
+    } else {
path: ql/src/java/org/apache/hadoop/hive/ql/DriverUtils.java
user: login: deniskuzZ
body: @difin, why did you add the `else` branch that overrides SessionState conf?
created_at: 2025-03-28T16:50:45Z
author_association: MEMBER
diff_hunk: @@ -1906,6 +1923,7 @@ public void setCurrentCatalog(String currentCatalog) {
   }
 
   public void close() throws IOException {
+    sessionStates.remove(this);
path: ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
user: login: deniskuzZ
body: Please add new line between and maybe add a comment "De-register session state"
created_at: 2025-03-28T16:54:58Z
author_association: MEMBER
diff_hunk: @@ -1969,6 +1987,38 @@ public void close() throws IOException {
     dynamicVars.clear();
   }
 
+  private static void cleanUpAllSessionStates() {
+    ExecutorService cleanupExecutor = Executors.newCachedThreadPool();
+    try {
+      CompletableFuture<Void> allCleanupTasks = CompletableFuture.allOf(sessionStates.stream()
+              .map(sessionState -> CompletableFuture.runAsync(() -> {
+                  try {
+                      LOG.info("Closing session state: {}", sessionState.getSessionId());
+                      sessionState.close();
+                  } catch (IOException e) {
+                      LOG.error("Problem closing session state", e);
+                  }
+              }, cleanupExecutor).exceptionally(e -> {
+                  LOG.error("Error closing session state", e);
+                  return null;
+              })).toArray(CompletableFuture[]::new));
+      try {
+        allCleanupTasks.get(60, TimeUnit.SECONDS);
+      } catch (Exception e) {
+        LOG.error("Failed to close all session states", e);
+      }
+    } finally {
+      cleanupExecutor.shutdown();
path: ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
user: login: deniskuzZ
body: why do you need a graceful shutdown if at this point all tasks either completed successfully or timed out?
created_at: 2025-03-28T17:18:27Z
author_association: MEMBER
diff_hunk: @@ -1969,6 +1987,38 @@ public void close() throws IOException {
     dynamicVars.clear();
   }
 
+  private static void cleanUpAllSessionStates() {
+    ExecutorService cleanupExecutor = Executors.newCachedThreadPool();
+    try {
+      CompletableFuture<Void> allCleanupTasks = CompletableFuture.allOf(sessionStates.stream()
+              .map(sessionState -> CompletableFuture.runAsync(() -> {
+                  try {
+                      LOG.info("Closing session state: {}", sessionState.getSessionId());
+                      sessionState.close();
+                  } catch (IOException e) {
+                      LOG.error("Problem closing session state", e);
path: ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
user: login: deniskuzZ
body: `throw new RuntimeException(e)` here to handle in the exceptionally
created_at: 2025-03-28T17:20:59Z
author_association: MEMBER
diff_hunk: @@ -38,28 +39,24 @@ public class KillTriggerActionHandler implements TriggerActionHandler<TezSession
   @Override
   public void applyAction(final Map<TezSessionState, Trigger> queriesViolated) {
     for (Map.Entry<TezSessionState, Trigger> entry : queriesViolated.entrySet()) {
-      switch (entry.getValue().getAction().getType()) {
-      case KILL_QUERY:
-        TezSessionState sessionState = entry.getKey();
-        String queryId = sessionState.getWmContext().getQueryId();
-        try {
-          UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-          SessionState ss = new SessionState(new HiveConf(), ugi.getShortUserName());
-          ss.setIsHiveServerQuery(true);
-          SessionState.start(ss);
-          KillQuery killQuery = sessionState.getKillQuery();
-          // if kill query is null then session might have been released to pool or closed already
-          if (killQuery != null) {
-            sessionState.getKillQuery().killQuery(queryId, entry.getValue().getViolationMsg(),
-                      sessionState.getConf());
-          }
-        } catch (HiveException|IOException e) {
-          LOG.warn("Unable to kill query {} for trigger violation", queryId);
+        if (entry.getValue().getAction().getType() == Action.Type.KILL_QUERY) {
+            TezSessionState sessionState = entry.getKey();
+            String queryId = sessionState.getWmContext().getQueryId();
+            try {
+                UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+                DriverUtils.setUpSessionState(new HiveConf(), ugi.getShortUserName());
path: ql/src/java/org/apache/hadoop/hive/ql/exec/tez/KillTriggerActionHandler.java
user: login: deniskuzZ
body: could we move `conf` to an instance var and initialise it in the constructor?
created_at: 2025-03-28T17:36:29Z
author_association: MEMBER
diff_hunk: @@ -1969,6 +1988,32 @@ public void close() throws IOException {
     dynamicVars.clear();
   }
 
+  private static void cleanUpAllSessionStates() {
+    ExecutorService cleanupExecutor = Executors.newCachedThreadPool();
+    try {
+      CompletableFuture<Void> allCleanupTasks = CompletableFuture.allOf(sessionStates.stream()
+              .map(sessionState -> CompletableFuture.runAsync(() -> {
+                  try {
+                      LOG.info("Closing session state: {}", sessionState.getSessionId());
+                      sessionState.close();
+                  } catch (IOException e) {
+                      throw new RuntimeException(e);
path: ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
user: login: deniskuzZ
body: minor: @Indhumathi27, seems that `CompletionException` is recommended here instead of `RuntimeException`. 
created_at: 2025-03-31T12:37:34Z
author_association: MEMBER
diff_hunk: @@ -167,12 +168,10 @@ private Loader(Hive db) {
 
     @Override
     public void run() {
-      SessionState ss = new SessionState(db.getConf());
-      ss.setIsHiveServerQuery(true); // All is served from HS2, we do not need e.g. Tez sessions
-      SessionState.start(ss);
       PerfLogger perfLogger = SessionState.getPerfLogger();
-      perfLogger.perfLogBegin(CLASS_NAME, PerfLogger.MATERIALIZED_VIEWS_REGISTRY_REFRESH);
       try {
+        DriverUtils.setUpSessionState(db.getConf());
path: ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
user: login: kasakrisz
body: nit: This refactor doesn't change the existing behavior. However the method `setUpSessionState` hides that we not just setup the SessionState but also start it. IMHO the original code emphasised this better.

A method name like `setUpAndStartSessionState` would also be better. 
WDYT?
created_at: 2025-04-01T06:33:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -167,12 +168,10 @@ private Loader(Hive db) {
 
     @Override
     public void run() {
-      SessionState ss = new SessionState(db.getConf());
-      ss.setIsHiveServerQuery(true); // All is served from HS2, we do not need e.g. Tez sessions
-      SessionState.start(ss);
       PerfLogger perfLogger = SessionState.getPerfLogger();
-      perfLogger.perfLogBegin(CLASS_NAME, PerfLogger.MATERIALIZED_VIEWS_REGISTRY_REFRESH);
       try {
+        DriverUtils.setUpSessionState(db.getConf());
path: ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMaterializedViewsRegistry.java
user: login: deniskuzZ
body: @Indhumathi27, please do not rename the original method with `doStart` param, others 2 are fine
  
created_at: 2025-04-01T07:40:52Z
author_association: MEMBER

PR comments:
user: login: Indhumathi27
created_at: 2025-02-24T05:17:35Z
body: @ayushtkn @okumin Can you help to review the PR.. thanks
author_association: CONTRIBUTOR
user: login: Indhumathi27
created_at: 2025-02-27T14:47:03Z
body: @deniskuzZ can you help to review
author_association: CONTRIBUTOR
user: login: deniskuzZ
created_at: 2025-03-18T16:43:51Z
body: Thanks, @Indhumathi27 ! If no other comments, I'll merge once we get a green build 
author_association: MEMBER
user: login: Indhumathi27
created_at: 2025-03-26T14:47:34Z
body: @deniskuzZ Adding Shutdown hook on SessionState.Start actually creating a memory overhead. while testing the change in a cluster, noticed considerably lot of shutdown hooks. And also, the shutdown hook tried closing a session which is already closed.
To overcome these issues, added a single ShutdownHook for the List of session states which are left unclosed. 
Handled this changes in the second commit of this PR.
Please help to review. Thanks
author_association: CONTRIBUTOR
user: login: Indhumathi27
created_at: 2025-03-30T16:43:22Z
body: @deniskuzZ Handled all the comments. Please check
author_association: CONTRIBUTOR
user: login: sonarqubecloud[bot]
created_at: 2025-04-01T17:10:37Z
body: ## [![Quality Gate Passed](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/checks/QualityGateBadge/qg-passed-20px.png 'Quality Gate Passed')](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=5649) **Quality Gate passed**  
Issues  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [4 New issues](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=5649&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/accepted-16px.png '') [0 Accepted issues](https://sonarcloud.io/project/issues?id=apache_hive&pullRequest=5649&issueStatuses=ACCEPTED)

Measures  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache_hive&pullRequest=5649&issueStatuses=OPEN,CONFIRMED&sinceLeakPeriod=true)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Coverage on New Code](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=5649&metric=new_coverage&view=list)  
![](https://sonarsource.github.io/sonarcloud-github-static-resources/v2/common/passed-16px.png '') [0.0% Duplication on New Code](https://sonarcloud.io/component_measures?id=apache_hive&pullRequest=5649&metric=new_duplicated_lines_density&view=list)  
  
[See analysis details on SonarQube Cloud](https://sonarcloud.io/dashboard?id=apache_hive&pullRequest=5649)


author_association: NONE
user: login: Indhumathi27
created_at: 2025-04-02T05:21:38Z
body: @deniskuzZ @kasakrisz Comments has been addressed. Please help to review and merge. Thanks
author_association: CONTRIBUTOR