Repository: AthenZ/athenz

Issue:
zms storage layer does not return multiple matches for the product id listing
each domain can be associated with a product id and the server provides the capability where you either have a unique product id in the system or allow multiple domains to be associated with the same product id.

however, when querying for the domain list with the given product id, the server only returns the first match instead of the full list.

the storage api must be updated to return a list of strings instead of just string.

Hints from issue comments:

Solution patch:
diff --git a/libs/java/server_common/src/main/java/com/yahoo/athenz/common/server/store/ObjectStoreConnection.java b/libs/java/server_common/src/main/java/com/yahoo/athenz/common/server/store/ObjectStoreConnection.java
index 27d2fd335..827345ad0 100644
--- a/libs/java/server_common/src/main/java/com/yahoo/athenz/common/server/store/ObjectStoreConnection.java
+++ b/libs/java/server_common/src/main/java/com/yahoo/athenz/common/server/store/ObjectStoreConnection.java
@@ -35,7 +35,7 @@ public interface ObjectStoreConnection extends Closeable {
     void commitChanges() throws ServerResourceException;
     void rollbackChanges() throws ServerResourceException;
     void close();
-    void setOperationTimeout(int opTimout);
+    void setOperationTimeout(int opTimeout);
     void setTagLimit(int domainLimit, int roleLimit, int groupLimit, int policyLimit, int serviceLimit);
 
     // Domain commands
@@ -47,9 +47,9 @@ public interface ObjectStoreConnection extends Closeable {
     long getDomainModTimestamp(String domainName) throws ServerResourceException;
     boolean updateDomainModTimestamp(String domainName) throws ServerResourceException;
     List<String> listDomains(String prefix, long modifiedSince) throws ServerResourceException;
-    String lookupDomainByProductId(int productId) throws ServerResourceException;
-    String lookupDomainByProductId(String productId) throws ServerResourceException;
-    String lookupDomainByCloudProvider(String provider, String value) throws ServerResourceException;
+    List<String> lookupDomainByProductId(int productId) throws ServerResourceException;
+    List<String> lookupDomainByProductId(String productId) throws ServerResourceException;
+    List<String> lookupDomainByCloudProvider(String provider, String value) throws ServerResourceException;
     Map<String, String> listDomainsByCloudProvider(String provider) throws ServerResourceException;
     List<String> lookupDomainByRole(String roleMember, String roleName) throws ServerResourceException;
     List<String> lookupDomainByBusinessService(String businessService) throws ServerResourceException;
diff --git a/libs/java/server_common/src/main/java/com/yahoo/athenz/common/server/store/impl/JDBCConnection.java b/libs/java/server_common/src/main/java/com/yahoo/athenz/common/server/store/impl/JDBCConnection.java
index 0ebeb8a26..636551713 100644
--- a/libs/java/server_common/src/main/java/com/yahoo/athenz/common/server/store/impl/JDBCConnection.java
+++ b/libs/java/server_common/src/main/java/com/yahoo/athenz/common/server/store/impl/JDBCConnection.java
@@ -1068,7 +1068,26 @@ public class JDBCConnection implements ObjectStoreConnection {
         }
     }
 
-    void verifyDomainProductIdUniqueness(final String name, Integer productNumber, final String caller) throws ServerResourceException {
+    void uniquenessCheck(final List<String> domains, final String domainName, final String label,
+            final String caller) throws ServerResourceException {
+
+        // if the domains list is null or empty then we're good
+
+        if (domains == null || domains.isEmpty()) {
+            return;
+        }
+
+        // if we have more than one domain or the single
+        // domain is different from our current domain
+        // then our value is not unique
+
+        if (domains.size() > 1 || !domains.get(0).equals(domainName)) {
+            throw requestError(caller, label + " is already assigned to domain: " + domains.get(0));
+        }
+    }
+
+    void verifyDomainProductIdUniqueness(final String domainName, Integer productNumber, final String caller)
+            throws ServerResourceException {
 
         if (productNumber == null || productNumber == 0) {
             return;
@@ -1076,13 +1095,11 @@ public class JDBCConnection implements ObjectStoreConnection {
         if (domainOptions != null && !domainOptions.getEnforceUniqueProductIds()) {
             return;
         }
-        final String domainName = lookupDomainByProductId(productNumber);
-        if (domainName != null && !domainName.equals(name)) {
-            throw requestError(caller, "Product Id: " + productNumber + " is already assigned to domain: " + domainName);
-        }
+        uniquenessCheck(lookupDomainByProductId(productNumber), domainName, "Product Id: " + productNumber, caller);
     }
 
-    void verifyDomainProductIdUniqueness(final String name, String productId, final String caller) throws ServerResourceException {
+    void verifyDomainProductIdUniqueness(final String domainName, String productId, final String caller)
+            throws ServerResourceException {
 
         if (StringUtil.isEmpty(productId)) {
             return;
@@ -1090,13 +1107,11 @@ public class JDBCConnection implements ObjectStoreConnection {
         if (domainOptions != null && !domainOptions.getEnforceUniqueProductIds()) {
             return;
         }
-        final String domainName = lookupDomainByProductId(productId);
-        if (domainName != null && !domainName.equals(name)) {
-            throw requestError(caller, "Product Id: " + productId + " is already assigned to domain: " + domainName);
-        }
+        uniquenessCheck(lookupDomainByProductId(productId), domainName, "Product Id: " + productId, caller);
     }
 
-    void verifyDomainAwsAccountUniqueness(final String name, final String account, final String caller) throws ServerResourceException {
+    void verifyDomainAwsAccountUniqueness(final String domainName, final String account, final String caller)
+            throws ServerResourceException {
 
         if (StringUtil.isEmpty(account)) {
             return;
@@ -1104,13 +1119,12 @@ public class JDBCConnection implements ObjectStoreConnection {
         if (domainOptions != null && !domainOptions.getEnforceUniqueAWSAccounts()) {
             return;
         }
-        final String domainName = lookupDomainByCloudProvider(ObjectStoreConnection.PROVIDER_AWS, account);
-        if (domainName != null && !domainName.equals(name)) {
-            throw requestError(caller, "Account Id: " + account + " is already assigned to domain: " + domainName);
-        }
+        uniquenessCheck(lookupDomainByCloudProvider(ObjectStoreConnection.PROVIDER_AWS, account), domainName,
+                "Account Id: " + account, caller);
     }
 
-    void verifyDomainAzureSubscriptionUniqueness(final String name, final String subscription, final String caller) throws ServerResourceException {
+    void verifyDomainAzureSubscriptionUniqueness(final String domainName, final String subscription,
+            final String caller) throws ServerResourceException {
 
         if (StringUtil.isEmpty(subscription)) {
             return;
@@ -1118,13 +1132,12 @@ public class JDBCConnection implements ObjectStoreConnection {
         if (domainOptions != null && !domainOptions.getEnforceUniqueAzureSubscriptions()) {
             return;
         }
-        final String domainName = lookupDomainByCloudProvider(ObjectStoreConnection.PROVIDER_AZURE, subscription);
-        if (domainName != null && !domainName.equals(name)) {
-            throw requestError(caller, "Subscription Id: " + subscription + " is already assigned to domain: " + domainName);
-        }
+        uniquenessCheck(lookupDomainByCloudProvider(ObjectStoreConnection.PROVIDER_AZURE, subscription), domainName,
+                "Subscription Id: " + subscription, caller);
     }
 
-    void verifyDomainGcpProjectUniqueness(final String name, final String project, final String caller) throws ServerResourceException {
+    void verifyDomainGcpProjectUniqueness(final String domainName, final String project, final String caller)
+            throws ServerResourceException {
 
         if (StringUtil.isEmpty(project)) {
             return;
@@ -1132,10 +1145,8 @@ public class JDBCConnection implements ObjectStoreConnection {
         if (domainOptions != null && !domainOptions.getEnforceUniqueGCPProjects()) {
             return;
         }
-        final String domainName = lookupDomainByCloudProvider(ObjectStoreConnection.PROVIDER_GCP, project);
-        if (domainName != null && !domainName.equals(name)) {
-            throw requestError(caller, "Project: " + project + " is already assigned to domain: " + domainName);
-        }
+        uniquenessCheck(lookupDomainByCloudProvider(ObjectStoreConnection.PROVIDER_GCP, project), domainName,
+                "Project: " + project, caller);
     }
 
     @Override
@@ -1343,41 +1354,41 @@ public class JDBCConnection implements ObjectStoreConnection {
     }
 
     @Override
-    public String lookupDomainByProductId(int productId) throws ServerResourceException {
+    public List<String> lookupDomainByProductId(int productId) throws ServerResourceException {
 
         final String caller = "lookupDomainByProductId";
-        String domainName = null;
+        List<String> domains = new ArrayList<>();
         try (PreparedStatement ps = con.prepareStatement(SQL_GET_DOMAIN_WITH_YPM_ID)) {
             ps.setInt(1, productId);
             try (ResultSet rs = executeQuery(ps, caller)) {
-                if (rs.next()) {
-                    domainName = rs.getString(1);
+                while (rs.next()) {
+                    domains.add(rs.getString(JDBCConsts.DB_COLUMN_NAME));
                 }
             }
         } catch (SQLException ex) {
             throw sqlError(ex, caller);
         }
-
-        return domainName;
+        Collections.sort(domains);
+        return domains;
     }
 
     @Override
-    public String lookupDomainByProductId(String productId) throws ServerResourceException {
+    public List<String> lookupDomainByProductId(String productId) throws ServerResourceException {
 
         final String caller = "lookupDomainByProductId";
-        String domainName = null;
+        List<String> domains = new ArrayList<>();
         try (PreparedStatement ps = con.prepareStatement(SQL_GET_DOMAIN_WITH_PRODUCT_ID)) {
             ps.setString(1, productId);
             try (ResultSet rs = executeQuery(ps, caller)) {
-                if (rs.next()) {
-                    domainName = rs.getString(1);
+                while (rs.next()) {
+                    domains.add(rs.getString(JDBCConsts.DB_COLUMN_NAME));
                 }
             }
         } catch (SQLException ex) {
             throw sqlError(ex, caller);
         }
-
-        return domainName;
+        Collections.sort(domains);
+        return domains;
     }
 
     String getCloudProviderLookupDomainSQLCommand(final String provider) {
@@ -1426,26 +1437,26 @@ public class JDBCConnection implements ObjectStoreConnection {
     }
 
     @Override
-    public String lookupDomainByCloudProvider(String provider, String value) throws ServerResourceException {
+    public List<String> lookupDomainByCloudProvider(String provider, String value) throws ServerResourceException {
 
         final String caller = "lookupDomainByCloudProvider";
         final String sqlCmd = getCloudProviderLookupDomainSQLCommand(provider);
         if (sqlCmd == null || value == null) {
-            return null;
+            return Collections.emptyList();
         }
-        String domainName = null;
+        List<String> domains = new ArrayList<>();
         try (PreparedStatement ps = con.prepareStatement(sqlCmd)) {
             ps.setString(1, value.trim());
             try (ResultSet rs = executeQuery(ps, caller)) {
-                if (rs.next()) {
-                    domainName = rs.getString(1);
+                while (rs.next()) {
+                    domains.add(rs.getString(JDBCConsts.DB_COLUMN_NAME));
                 }
             }
         } catch (SQLException ex) {
             throw sqlError(ex, caller);
         }
-
-        return domainName;
+        Collections.sort(domains);
+        return domains;
     }
 
     @Override
@@ -1454,7 +1465,7 @@ public class JDBCConnection implements ObjectStoreConnection {
         final String caller = "listDomainByCloudProvider";
         final String sqlCmd = getCloudProviderListDomainsSQLCommand(provider);
         if (sqlCmd == null) {
-            return null;
+            return Collections.emptyMap();
         }
         final String columnName = getCloudProviderColumnName(provider);
         Map<String, String> domains = new HashMap<>();
diff --git a/servers/zms/src/main/java/com/yahoo/athenz/zms/DBService.java b/servers/zms/src/main/java/com/yahoo/athenz/zms/DBService.java
index fd2f79992..d36a58ee6 100644
--- a/servers/zms/src/main/java/com/yahoo/athenz/zms/DBService.java
+++ b/servers/zms/src/main/java/com/yahoo/athenz/zms/DBService.java
@@ -3486,9 +3486,9 @@ public class DBService implements RolesProvider, DomainProvider {
 
         DomainList domList = new DomainList();
         try (ObjectStoreConnection con = store.getConnection(true, false)) {
-            final String domain = con.lookupDomainByCloudProvider(provider, value);
-            if (domain != null) {
-                domList.setNames(Collections.singletonList(domain));
+            final List<String> domains = con.lookupDomainByCloudProvider(provider, value);
+            if (!domains.isEmpty()) {
+                domList.setNames(domains);
             }
         } catch (ServerResourceException ex) {
             throw ZMSUtils.error(ex);
@@ -3512,9 +3512,9 @@ public class DBService implements RolesProvider, DomainProvider {
 
         DomainList domList = new DomainList();
         try (ObjectStoreConnection con = store.getConnection(true, false)) {
-            String domain = con.lookupDomainByProductId(productId);
-            if (domain != null) {
-                domList.setNames(Collections.singletonList(domain));
+            final List<String> domains = con.lookupDomainByProductId(productId);
+            if (!domains.isEmpty()) {
+                domList.setNames(domains);
             }
         } catch (ServerResourceException ex) {
             throw ZMSUtils.error(ex);
@@ -3526,9 +3526,9 @@ public class DBService implements RolesProvider, DomainProvider {
 
         DomainList domList = new DomainList();
         try (ObjectStoreConnection con = store.getConnection(true, false)) {
-            String domain = con.lookupDomainByProductId(productId);
-            if (domain != null) {
-                domList.setNames(Collections.singletonList(domain));
+            final List<String> domains = con.lookupDomainByProductId(productId);
+            if (!domains.isEmpty()) {
+                domList.setNames(domains);
             }
         } catch (ServerResourceException ex) {
             throw ZMSUtils.error(ex);
diff --git a/ui/package.json b/ui/package.json
index 766470672..a0a1eed7e 100644
--- a/ui/package.json
+++ b/ui/package.json
@@ -54,6 +54,19 @@
         "@emotion/react": "11.10.4",
         "@emotion/server": "11.10.0",
         "@emotion/styled": "11.10.4",
+        "@grpc/grpc-js": "^1.9.0",
+        "@opentelemetry/api": "^1.9.0",
+        "@opentelemetry/auto-instrumentations-node": "^0.53.0",
+        "@opentelemetry/exporter-logs-otlp-grpc": "^0.53.0",
+        "@opentelemetry/exporter-metrics-otlp-grpc": "^0.53.0",
+        "@opentelemetry/exporter-trace-otlp-grpc": "^0.53.0",
+        "@opentelemetry/resources": "^1.28.0",
+        "@opentelemetry/sdk-logs": "^0.53.0",
+        "@opentelemetry/sdk-metrics": "^1.28.0",
+        "@opentelemetry/sdk-node": "^0.53.0",
+        "@opentelemetry/sdk-trace-node": "^1.28.0",
+        "@opentelemetry/semantic-conventions": "^1.28.0",
+        "@opentelemetry/instrumentation-runtime-node": "^0.22.0",
         "@redux-devtools/extension": "3.2.3",
         "@reduxjs/toolkit": "1.8.5",
         "async": "3.2.4",
diff --git a/ui/pm2.config.js b/ui/pm2.config.js
index 449ca2536..1cf91a4d9 100644
--- a/ui/pm2.config.js
+++ b/ui/pm2.config.js
@@ -22,6 +22,8 @@ module.exports = {
             name: 'athenz-ui',
             cwd: APP_DIR,
             script: require(path.join(APP_DIR, 'package.json')).main,
+            // Load OpenTelemetry instrumentation BEFORE app starts
+            node_args: '--import ./src/opentelemetry/instrumentation.mjs',
             env_production: {
                 NODE_ENV: 'production',
             },
diff --git a/ui/src/opentelemetry/instrumentation.mjs b/ui/src/opentelemetry/instrumentation.mjs
new file mode 100644
index 000000000..e4a848794
--- /dev/null
+++ b/ui/src/opentelemetry/instrumentation.mjs
@@ -0,0 +1,132 @@
+/*instrumentation.mjs*/
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc';
+import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-grpc';
+import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc';
+import { BatchLogRecordProcessor } from '@opentelemetry/sdk-logs';
+import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
+import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
+import { credentials } from '@grpc/grpc-js';
+import { readFileSync } from 'fs';
+import { RuntimeNodeInstrumentation } from '@opentelemetry/instrumentation-runtime-node';
+import { Resource } from '@opentelemetry/resources';
+import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
+
+const STATIC_ASSET_EXTENSIONS = ['.js', '.css', '.svg', '.js.map', '.webp'];
+
+const ignoreStaticAssets = (request = {}) => {
+    const url = request.url?.toLowerCase?.() || '';
+    return STATIC_ASSET_EXTENSIONS.some((ext) => url.endsWith(ext));
+};
+
+const updateSpanName = (span, request = {}) => {
+    const rawUrl = request.url || '';
+    const path = rawUrl.split('?')[0] || '/';
+    const method = request.method || 'UNKNOWN';
+    span.updateName(`${method} ${path}`);
+};
+
+// Setup gRPC credentials with TLS
+let grpcCredentials;
+let exporterOptions;
+const allowInsecureFallback =
+    process.env.OTEL_ALLOW_INSECURE_FALLBACK === 'true';
+
+const caPath = process.env.OTEL_EXPORTER_OTLP_CA_CERTIFICATE;
+
+if (caPath) {
+    try {
+        const caCert = readFileSync(caPath);
+        grpcCredentials = credentials.createSsl(caCert);
+        exporterOptions = { credentials: grpcCredentials };
+    } catch (err) {
+        console.error('[OTEL] Failed to read CA certificate:', err.message);
+        console.error(
+            '[OTEL] OTEL_EXPORTER_OTLP_CA_CERTIFICATE is set but invalid. Shutting down.'
+        );
+        throw err;
+    }
+} else {
+    if (allowInsecureFallback) {
+        console.warn(
+            '[OTEL] OTEL_EXPORTER_OTLP_CA_CERTIFICATE not set, OTEL_ALLOW_INSECURE_FALLBACK=true, using insecure OTLP connection'
+        );
+        grpcCredentials = credentials.createInsecure();
+        exporterOptions = { credentials: grpcCredentials };
+    } else {
+        console.error(
+            '[OTEL] OTEL_EXPORTER_OTLP_CA_CERTIFICATE not set and insecure fallback is not allowed. Shutting down.'
+        );
+        throw new Error(
+            '[OTEL] No CA certificate configured and OTEL_ALLOW_INSECURE_FALLBACK is not "true".'
+        );
+    }
+}
+
+// Initialize OpenTelemetry SDK
+const sdk = new NodeSDK({
+    resource: new Resource({
+        [SemanticResourceAttributes.SERVICE_NAME]: 'athenz-ui',
+        [SemanticResourceAttributes.SERVICE_NAMESPACE]: 'AthenzUI',
+        [SemanticResourceAttributes.DEPLOYMENT_ENVIRONMENT]:
+            process.env.ENVIRONMENT || 'local',
+    }),
+
+    traceExporter: new OTLPTraceExporter(exporterOptions),
+    logRecordProcessor: new BatchLogRecordProcessor(
+        new OTLPLogExporter(exporterOptions)
+    ),
+    metricReader: new PeriodicExportingMetricReader({
+        exporter: new OTLPMetricExporter(exporterOptions),
+        exportIntervalMillis: 60000,
+    }),
+    instrumentations: [
+        getNodeAutoInstrumentations({
+            '@opentelemetry/instrumentation-aws-sdk': { enabled: false },
+            '@opentelemetry/instrumentation-http': {
+                enabled: true,
+                // Ignore static assets like JS/CSS so dashboards focus on real endpoints
+                ignoreIncomingRequestHook: ignoreStaticAssets,
+                // Improve span names for better traces / metrics labels
+                requestHook: (span, request) => {
+                    updateSpanName(span, request);
+                },
+            },
+        }),
+        new RuntimeNodeInstrumentation({
+            enabled: true,
+            exportFloats: true,
+            memory: { enabled: true },
+            cpu: { enabled: true },
+            eventLoop: { enabled: true },
+        }),
+    ],
+});
+
+// -----------------------------
+// Start + graceful shutdown
+// -----------------------------
+
+try {
+    sdk.start();
+    console.log('[OTEL] OpenTelemetry initialized');
+} catch (err) {
+    console.error('[OTEL] Failed to start OpenTelemetry SDK:', err.message);
+    process.exit(1);
+}
+
+const gracefulShutdown = () => {
+    console.log('[OTEL] Shutting down OpenTelemetry SDK...');
+    sdk.shutdown()
+        .then(() => {
+            console.log('[OTEL] SDK shut down successfully');
+            process.exit(0);
+        })
+        .catch((error) => {
+            console.error('[OTEL] Error shutting down SDK', error);
+            process.exit(1);
+        });
+};
+
+process.on('SIGTERM', gracefulShutdown);
+process.on('SIGINT', gracefulShutdown);



Test patch:
diff --git a/libs/java/server_common/src/test/java/com/yahoo/athenz/common/server/store/impl/JDBCConnectionTest.java b/libs/java/server_common/src/test/java/com/yahoo/athenz/common/server/store/impl/JDBCConnectionTest.java
index 606c4ea88..b96f6e865 100644
--- a/libs/java/server_common/src/test/java/com/yahoo/athenz/common/server/store/impl/JDBCConnectionTest.java
+++ b/libs/java/server_common/src/test/java/com/yahoo/athenz/common/server/store/impl/JDBCConnectionTest.java
@@ -3716,7 +3716,7 @@ public class JDBCConnectionTest {
 
         List<Policy> policies = jdbcConn.deleteAssumeRoleAssertions(
                 "tenant", "provider", "admin-role");
-        assertEquals(2, policies.size());
+        assertEquals(policies.size(), 2);
 
         /* 3 Verify parameter binding / execution counts */
         Mockito.verify(mockPrepStmt, times(2)).setString(1, "tenant");                // SELECT & DELETE
@@ -3760,7 +3760,7 @@ public class JDBCConnectionTest {
             jdbcConn.deleteAssumeRoleAssertions("tenant", "provider", "admin-role");
             fail();
         } catch (ServerResourceException ex) {
-            assertEquals(ServerResourceException.INTERNAL_SERVER_ERROR, ex.getCode());
+            assertEquals(ex.getCode(), ServerResourceException.INTERNAL_SERVER_ERROR);
         }
         jdbcConn.close();
     }
@@ -3786,7 +3786,7 @@ public class JDBCConnectionTest {
             jdbcConn.deleteAssumeRoleAssertions("tenant", "provider", "admin-role");
             fail();
         } catch (ServerResourceException ex) {
-            assertEquals(ServerResourceException.INTERNAL_SERVER_ERROR, ex.getCode());
+            assertEquals(ex.getCode(), ServerResourceException.INTERNAL_SERVER_ERROR);
         }
         jdbcConn.close();
     }
@@ -3822,7 +3822,7 @@ public class JDBCConnectionTest {
             jdbcConn.deleteAssumeRoleAssertions("tenant", "provider", "admin-role");
             fail(); // We must never get here.
         } catch (ServerResourceException ex) {
-            assertEquals(ServerResourceException.INTERNAL_SERVER_ERROR, ex.getCode());
+            assertEquals(ex.getCode(), ServerResourceException.INTERNAL_SERVER_ERROR);
         }
 
         jdbcConn.close();
@@ -7077,7 +7077,7 @@ public class JDBCConnectionTest {
     public void testVerifyDomainSubscriptionUniquenessFail() throws Exception {
 
         Mockito.when(mockResultSet.next()).thenReturn(true).thenReturn(false);
-        Mockito.doReturn("iaas.athenz.ci").when(mockResultSet).getString(1);
+        Mockito.doReturn("iaas.athenz.ci").when(mockResultSet).getString("name");
 
         JDBCConnection jdbcConn = new JDBCConnection(mockConn, true);
         try {
@@ -7117,7 +7117,7 @@ public class JDBCConnectionTest {
     public void testVerifyDomainProjectUniquenessFail() throws Exception {
 
         Mockito.when(mockResultSet.next()).thenReturn(true).thenReturn(false);
-        Mockito.doReturn("iaas.athenz.ci").when(mockResultSet).getString(1);
+        Mockito.doReturn("iaas.athenz.ci").when(mockResultSet).getString("name");
 
         JDBCConnection jdbcConn = new JDBCConnection(mockConn, true);
         try {
@@ -7157,7 +7157,7 @@ public class JDBCConnectionTest {
     public void testVerifyDomainAccountUniquenessPass() throws Exception {
 
         Mockito.when(mockResultSet.next()).thenReturn(true).thenReturn(false);
-        Mockito.doReturn("iaas.athenz").when(mockResultSet).getString(1);
+        Mockito.doReturn("iaas.athenz").when(mockResultSet).getString("name");
 
         JDBCConnection jdbcConn = new JDBCConnection(mockConn, true);
         jdbcConn.verifyDomainAwsAccountUniqueness("iaas.athenz", "12345", "unitTest");
@@ -7178,7 +7178,7 @@ public class JDBCConnectionTest {
     public void testVerifyDomainAccountUniquenessFail() throws Exception {
 
         Mockito.when(mockResultSet.next()).thenReturn(true).thenReturn(false);
-        Mockito.doReturn("iaas.athenz.ci").when(mockResultSet).getString(1);
+        Mockito.doReturn("iaas.athenz.ci").when(mockResultSet).getString("name");
 
         JDBCConnection jdbcConn = new JDBCConnection(mockConn, true);
         try {
@@ -7218,7 +7218,7 @@ public class JDBCConnectionTest {
     public void testVerifyDomainProductIdNumberUniquenessPass() throws Exception {
 
         Mockito.when(mockResultSet.next()).thenReturn(true).thenReturn(false);
-        Mockito.doReturn("iaas.athenz").when(mockResultSet).getString(1);
+        Mockito.doReturn("iaas.athenz").when(mockResultSet).getString("name");
 
         JDBCConnection jdbcConn = new JDBCConnection(mockConn, true);
         jdbcConn.verifyDomainProductIdUniqueness("iaas.athenz", 1001, "unitTest");
@@ -7239,7 +7239,7 @@ public class JDBCConnectionTest {
     public void testVerifyDomainProductIdNumberUniquenessFail() throws Exception {
 
         Mockito.when(mockResultSet.next()).thenReturn(true).thenReturn(false);
-        Mockito.doReturn("iaas.athenz.ci").when(mockResultSet).getString(1);
+        Mockito.doReturn("iaas.athenz.ci").when(mockResultSet).getString("name");
 
         JDBCConnection jdbcConn = new JDBCConnection(mockConn, true);
         try {
@@ -7279,7 +7279,7 @@ public class JDBCConnectionTest {
     public void testVerifyDomainProductIdUniquenessPass() throws Exception {
 
         Mockito.when(mockResultSet.next()).thenReturn(true).thenReturn(false);
-        Mockito.doReturn("iaas.athenz").when(mockResultSet).getString(1);
+        Mockito.doReturn("iaas.athenz").when(mockResultSet).getString("name");
 
         JDBCConnection jdbcConn = new JDBCConnection(mockConn, true);
         jdbcConn.verifyDomainProductIdUniqueness("iaas.athenz", "abcd-1001", "unitTest");
@@ -7300,7 +7300,7 @@ public class JDBCConnectionTest {
     public void testVerifyDomainProductIdUniquenessFail() throws Exception {
 
         Mockito.when(mockResultSet.next()).thenReturn(true).thenReturn(false);
-        Mockito.doReturn("iaas.athenz.ci").when(mockResultSet).getString(1);
+        Mockito.doReturn("iaas.athenz.ci").when(mockResultSet).getString("name");
 
         JDBCConnection jdbcConn = new JDBCConnection(mockConn, true);
         try {
@@ -7323,10 +7323,10 @@ public class JDBCConnectionTest {
     public void testLookupDomainByAwsAccount() throws Exception {
 
         Mockito.when(mockResultSet.next()).thenReturn(true).thenReturn(false);
-        Mockito.doReturn("iaas.athenz").when(mockResultSet).getString(1);
+        Mockito.doReturn("iaas.athenz").when(mockResultSet).getString("name");
 
         JDBCConnection jdbcConn = new JDBCConnection(mockConn, true);
-        final String domainName = jdbcConn.lookupDomainByCloudProvider(ObjectStoreConnection.PROVIDER_AWS, "1234");
+        final String domainName = jdbcConn.lookupDomainByCloudProvider(ObjectStoreConnection.PROVIDER_AWS, "1234").get(0);
         assertEquals(domainName, "iaas.athenz");
         jdbcConn.close();
     }
@@ -7335,10 +7335,10 @@ public class JDBCConnectionTest {
     public void testLookupDomainByAzureSubscription() throws Exception {
 
         Mockito.when(mockResultSet.next()).thenReturn(true).thenReturn(false);
-        Mockito.doReturn("iaas.athenz").when(mockResultSet).getString(1);
+        Mockito.doReturn("iaas.athenz").when(mockResultSet).getString("name");
 
         JDBCConnection jdbcConn = new JDBCConnection(mockConn, true);
-        final String domainName = jdbcConn.lookupDomainByCloudProvider(ObjectStoreConnection.PROVIDER_AZURE, "azure");
+        final String domainName = jdbcConn.lookupDomainByCloudProvider(ObjectStoreConnection.PROVIDER_AZURE, "azure").get(0);
         assertEquals(domainName, "iaas.athenz");
         jdbcConn.close();
     }
@@ -7347,10 +7347,10 @@ public class JDBCConnectionTest {
     public void testLookupDomainByGcpProject() throws Exception {
 
         Mockito.when(mockResultSet.next()).thenReturn(true).thenReturn(false);
-        Mockito.doReturn("iaas.athenz").when(mockResultSet).getString(1);
+        Mockito.doReturn("iaas.athenz").when(mockResultSet).getString("name");
 
         JDBCConnection jdbcConn = new JDBCConnection(mockConn, true);
-        final String domainName = jdbcConn.lookupDomainByCloudProvider(ObjectStoreConnection.PROVIDER_GCP, "gcp");
+        final String domainName = jdbcConn.lookupDomainByCloudProvider(ObjectStoreConnection.PROVIDER_GCP, "gcp").get(0);
         assertEquals(domainName, "iaas.athenz");
         jdbcConn.close();
     }
@@ -7361,9 +7361,9 @@ public class JDBCConnectionTest {
         Mockito.when(mockPrepStmt.executeQuery()).thenThrow(new SQLException("failed operation", "state", 1001));
 
         JDBCConnection jdbcConn = new JDBCConnection(mockConn, true);
-        assertNull(jdbcConn.lookupDomainByCloudProvider(null, "iaas.athenz"));
-        assertNull(jdbcConn.lookupDomainByCloudProvider("unknown", "iaas.athenz"));
-        assertNull(jdbcConn.lookupDomainByCloudProvider("aws", null));
+        assertTrue(jdbcConn.lookupDomainByCloudProvider(null, "iaas.athenz").isEmpty());
+        assertTrue(jdbcConn.lookupDomainByCloudProvider("unknown", "iaas.athenz").isEmpty());
+        assertTrue(jdbcConn.lookupDomainByCloudProvider("aws", null).isEmpty());
 
         try {
             jdbcConn.lookupDomainByCloudProvider("aws", "iaas.athenz");
@@ -7406,10 +7406,10 @@ public class JDBCConnectionTest {
     public void testLookupDomainByProductIdNumber() throws Exception {
 
         Mockito.when(mockResultSet.next()).thenReturn(true).thenReturn(false);
-        Mockito.doReturn("iaas.athenz").when(mockResultSet).getString(1);
+        Mockito.doReturn("iaas.athenz").when(mockResultSet).getString("name");
 
         JDBCConnection jdbcConn = new JDBCConnection(mockConn, true);
-        String domainName = jdbcConn.lookupDomainByProductId(1001);
+        String domainName = jdbcConn.lookupDomainByProductId(1001).get(0);
         assertEquals(domainName, "iaas.athenz");
         jdbcConn.close();
     }
@@ -7433,10 +7433,10 @@ public class JDBCConnectionTest {
     public void testLookupDomainByProductId() throws Exception {
 
         Mockito.when(mockResultSet.next()).thenReturn(true).thenReturn(false);
-        Mockito.doReturn("iaas.athenz").when(mockResultSet).getString(1);
+        Mockito.doReturn("iaas.athenz").when(mockResultSet).getString("name");
 
         JDBCConnection jdbcConn = new JDBCConnection(mockConn, true);
-        String domainName = jdbcConn.lookupDomainByProductId("abcd-1001");
+        String domainName = jdbcConn.lookupDomainByProductId("abcd-1001").get(0);
         assertEquals(domainName, "iaas.athenz");
         jdbcConn.close();
     }
@@ -7920,7 +7920,7 @@ public class JDBCConnectionTest {
     public void testListDomainsByCloudProviderUnknown() throws Exception {
 
         JDBCConnection jdbcConn = new JDBCConnection(mockConn, true);
-        assertNull(jdbcConn.listDomainsByCloudProvider("unknown"));
+        assertTrue(jdbcConn.listDomainsByCloudProvider("unknown").isEmpty());
         jdbcConn.close();
     }
 
diff --git a/servers/zms/src/test/java/com/yahoo/athenz/zms/ZMSMetaAttributeTest.java b/servers/zms/src/test/java/com/yahoo/athenz/zms/ZMSMetaAttributeTest.java
index 98b254bd4..c9a310c7e 100644
--- a/servers/zms/src/test/java/com/yahoo/athenz/zms/ZMSMetaAttributeTest.java
+++ b/servers/zms/src/test/java/com/yahoo/athenz/zms/ZMSMetaAttributeTest.java
@@ -1541,4 +1541,38 @@ public class ZMSMetaAttributeTest {
         // restore the original meta store
         zmsImpl.domainMetaStore = savedMetaStore;
     }
+
+    @Test
+    public void testMultipleDomainsWithSameProductId() {
+        System.setProperty("athenz.zms.enforce_unique_product_ids", "false");
+        ZMSImpl zmsTest = zmsTestInitializer.zmsInit();
+
+        RsrcCtxWrapper ctx = zmsTestInitializer.getMockDomRsrcCtx();
+        final String auditRef = zmsTestInitializer.getAuditRef();
+        final String productId = "same-product-id";
+
+        final String domainName1 = "athenz-domain1-same-productid";
+        TopLevelDomain dom1 = zmsTestInitializer.createTopLevelDomainObject(domainName1,
+                "Test Domain1", "testOrg", zmsTestInitializer.getAdminUser());
+        dom1.setProductId(productId);
+        zmsTest.postTopLevelDomain(ctx, auditRef, null, dom1);
+
+        final String domainName2 = "athenz-domain2-same-productid";
+        TopLevelDomain dom2 = zmsTestInitializer.createTopLevelDomainObject(domainName2
+                , "Test Domain2", "testOrg", zmsTestInitializer.getAdminUser());
+        dom2.setProductId(productId);
+        zmsTest.postTopLevelDomain(ctx, auditRef, null, dom2);
+
+        DomainList domainList = zmsTest.getDomainList(ctx, null, null, null, null, null, null, null, null,
+                null, null, null, null, null, productId, null);
+        assertNotNull(domainList);
+        assertEquals(domainList.getNames().size(), 2);
+        assertTrue(domainList.getNames().contains(domainName1));
+        assertTrue(domainList.getNames().contains(domainName2));
+
+        zmsTest.deleteTopLevelDomain(ctx, domainName1, auditRef, null);
+        zmsTest.deleteTopLevelDomain(ctx, domainName2, auditRef, null);
+
+        System.clearProperty("athenz.zms.enforce_unique_product_ids");
+    }
 }