Repository: stardog-union/pystardog (language: python)

Issue:
standby node is failing to join the cluster on tests
From stardog 8+, now we see the standby node can't join the cluster. needs further investigation. For now the tests are skipped


Discussion / hints:

Fix patch:
diff --git a/docker-compose.cluster.yml b/docker-compose.cluster.yml
index 788f2b8..3af9b4a 100644
--- a/docker-compose.cluster.yml
+++ b/docker-compose.cluster.yml
@@ -43,23 +43,20 @@ services:
     depends_on:
       - zoo1
 
-
-# Disabled as we need to figure out why these are failing
-# https://github.com/stardog-union/pystardog/issues/111
-#  sdstandby:
-#    build:
-#      context: dockerfiles/
-#      dockerfile: dockerfile-stardog
-#      args:
-#        - TAG=${STARDOG_TAG}
-#        - NODE_TYPE=standby
-#        - SSH_USER=${SSH_USER}
-#        - SSH_PASS=${SSH_PASS}
-#    container_name: ${STARDOG_HOSTNAME_STANDBY}
-#    entrypoint: ["/bin/bash", "-c"]
-#    command: ["/var/start-standby.sh"]
-#    depends_on:
-#      - zoo1
+  sdstandby:
+    build:
+      context: dockerfiles/
+      dockerfile: dockerfile-stardog
+      args:
+        - TAG=${STARDOG_TAG}
+        - NODE_TYPE=standby
+        - SSH_USER=${SSH_USER}
+        - SSH_PASS=${SSH_PASS}
+    container_name: ${STARDOG_HOSTNAME_STANDBY}
+    entrypoint: ["/bin/bash", "-c"]
+    command: ["/var/start-standby.sh"]
+    depends_on:
+      - zoo1
 
   sdcache1:
     build:
diff --git a/dockerfiles/dockerfile-stardog b/dockerfiles/dockerfile-stardog
index 6eec381..f661d18 100644
--- a/dockerfiles/dockerfile-stardog
+++ b/dockerfiles/dockerfile-stardog
@@ -6,7 +6,6 @@ RUN wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-${
   && unzip mysql-connector-java-${CONNECTOR_VERSION}.zip \
   && mv mysql-connector-java-${CONNECTOR_VERSION}/mysql-connector-java-${CONNECTOR_VERSION}.jar /tmp/mysql-connector-java.jar
 
-
 FROM stardog/stardog
 
 ARG NODE_TYPE
diff --git a/dockerfiles/stardog.node.properties b/dockerfiles/stardog.node.properties
index cd2bfa8..7757209 100644
--- a/dockerfiles/stardog.node.properties
+++ b/dockerfiles/stardog.node.properties
@@ -5,3 +5,5 @@ pack.enabled=true
 #pack.node.address=196.69.68.4
 # the connection string for ZooKeeper where cluster state is stored
 pack.zookeeper.address=zoo1:2181
+# So it doesn't conflict with other tests.
+catalog.auto.create.db=false
diff --git a/dockerfiles/stardog.single.properties b/dockerfiles/stardog.single.properties
index d2b29d6..15e6693 100644
--- a/dockerfiles/stardog.single.properties
+++ b/dockerfiles/stardog.single.properties
@@ -1,2 +1,2 @@
 # Empty as we currently don't need specific properties for single node testing
-# we still need an existing property file so that the Dockerfile doesn't break
\ No newline at end of file
+# we still need an existing property file so that the Dockerfile doesn't break
diff --git a/dockerfiles/stardog.standby.properties b/dockerfiles/stardog.standby.properties
index a20b860..489428f 100644
--- a/dockerfiles/stardog.standby.properties
+++ b/dockerfiles/stardog.standby.properties
@@ -10,3 +10,7 @@ pack.zookeeper.address=zoo1:2181
 # https://docs.stardog.com/cluster/operating-the-cluster/standby-nodes
 pack.standby=true
 pack.standby.node.sync.interval=5m
+
+# otherwise we try to join a cluster with databases into an empty cluster, which turns into an error.
+# an alternative is to start the server with --force-join.
+catalog.auto.create.db=false
\ No newline at end of file
diff --git a/dockerfiles/start-standby.sh b/dockerfiles/start-standby.sh
index 88cca83..51e7d81 100755
--- a/dockerfiles/start-standby.sh
+++ b/dockerfiles/start-standby.sh
@@ -22,6 +22,7 @@ function wait_for_cluster_ready(){
     )
 
 }
+
 wait_for_cluster_ready
 echo "Main cluster ready, starting standby node"
 /var/start.sh
diff --git a/utils/wait.sh b/utils/wait.sh
index a01524b..494399c 100755
--- a/utils/wait.sh
+++ b/utils/wait.sh
@@ -12,7 +12,7 @@ function wait_for_start {
     do
       if [[ ${COUNT} -gt 50 ]]; then
           echo "Failed to start Stardog cluster on time"
-          return 1;
+          exit 1;
       fi
       COUNT=$(expr 1 + ${COUNT} )
       sleep 5
@@ -43,12 +43,13 @@ function wait_for_standby_node {
     do
       if [[ ${COUNT} -gt 50 ]]; then
           echo "Failed to start Stardog cluster on time"
-          return 1;
+          exit 1;
       fi
       COUNT=$(expr 1 + ${COUNT} )
       sleep 5
       # wait for standby node to be ready. standby nodes needs to wait for main cluster first.
-      status_code=$(curl -o /dev/null -s -w "%{http_code}\n" http://${HOST}:${PORT}/admin/healthcheck)
+      # Recall that checking healthcheck endpoint will return 503 since the node is still standby, and not part of a cluster, hence we check aliveness
+      status_code=$(curl -o /dev/null -s -w "%{http_code}\n" http://${HOST}:${PORT}/admin/alive)
       if [[ $status_code -eq 200 ]]; then break; fi
 
     done
@@ -67,7 +68,7 @@ function wait_for_start_single_node {
     do
       if [[ ${COUNT} -gt 50 ]]; then
           echo "Failed to start Stardog server on time"
-          return 1;
+          exit 1;
       fi
       COUNT=$(expr 1 + ${COUNT} )
       sleep 5


Test patch:
diff --git a/test/test_admin.py b/test/test_admin.py
index c20dcbc..f73d58d 100644
--- a/test/test_admin.py
+++ b/test/test_admin.py
@@ -48,13 +48,23 @@ def wait_standby_node_to_join(admin):
     )
     retries = 0
     while True:
-        if standby_node_ip in admin.cluster_info()["nodes"]:
-            return
-        else:
-            retries += 1
-            sleep(10)
-            if retries >= 50:
-                raise Exception("Took too long for standby node to join the cluster")
+        try:
+            if standby_node_ip in admin.cluster_info()["nodes"]:
+                print(f"current nodes: {admin.cluster_info()['nodes']}")
+                break
+            else:
+                print(
+                    "http call did not fail, but node is still not listed in cluster info"
+                )
+        except Exception as e:
+            print(
+                f"An exception ocurred while connecting to the standby node: {e}, will keep retrying"
+            )
+        print(f"retries for now: {retries}")
+        retries += 1
+        sleep(20)
+        if retries >= 50:
+            raise Exception("Took too long for standby node to join the cluster")
 
 
 def count_records(bd_name, conn_string):
@@ -109,6 +119,7 @@ def admin(conn_string):
 
         # alternatively we could warn somewhere (README, when running the tests, or other places)
         # that this are not intended to run against any other stardog deployment that is not a clean one.
+
         databases = admin.databases()
         if databases:
             for db in admin.databases():
@@ -158,6 +169,8 @@ def virtual_graph_music(admin, datasource_music):
 ##############
 # Admin tests#
 ##############
+
+
 @pytest.mark.skip(
     reason="Implementation is not well documented, https://stardog.atlassian.net/browse/PLAT-2946"
 )
@@ -187,25 +200,15 @@ def test_coordinator_check(admin, conn_string):
         assert admin_coordinator_check.cluster_coordinator_check()
 
 
-@pytest.mark.skip(
-    reason="Stardog 8 made this test fail, needs further investigation: https://github.com/stardog-union/pystardog/issues/111"
-)
-def test_cluster_standby(cluster_standby_node_conn_string):
+# We should pass a standby admin object instead of a connection string.
+def test_cluster_standby(admin, cluster_standby_node_conn_string):
 
     with stardog.admin.Admin(**cluster_standby_node_conn_string) as admin_standby:
-
         assert admin_standby.standby_node_pause(pause=True)
         assert admin_standby.standby_node_pause_status()["STATE"] == "PAUSED"
         assert admin_standby.standby_node_pause(pause=False)
         assert admin_standby.standby_node_pause_status()["STATE"] == "WAITING"
 
-        standby_nodes = admin_standby.cluster_list_standby_nodes()
-        node_id = standby_nodes["standbynodes"][0]
-        # removes a standby node from the registry, i.e from syncing with the rest of the cluster.
-        admin_standby.cluster_revoke_standby_access(standby_nodes["standbynodes"][0])
-        standby_nodes_revoked = admin_standby.cluster_list_standby_nodes()
-        assert node_id not in standby_nodes_revoked["standbynodes"]
-
         # Join a standby node is still allowed even if it's not part of the registry.
         admin_standby.cluster_join()
         wait_standby_node_to_join(admin_standby)
@@ -216,6 +219,13 @@ def test_cluster_standby(cluster_standby_node_conn_string):
         )
         assert standby_node_info in admin_standby.cluster_info()["nodes"]
 
+        standby_nodes = admin_standby.cluster_list_standby_nodes()
+        node_id = standby_nodes["standbynodes"][0]
+        # removes a standby node from the registry, i.e from syncing with the rest of the cluster.
+        admin_standby.cluster_revoke_standby_access(standby_nodes["standbynodes"][0])
+        standby_nodes_revoked = admin_standby.cluster_list_standby_nodes()
+        assert node_id not in standby_nodes_revoked["standbynodes"]
+
 
 def test_get_server_metrics(admin):
     assert "dbms.storage.levels" in admin.get_server_metrics()
@@ -282,12 +292,12 @@ def test_backup_all(admin):
 
 # DEPRECATED, test moved to test_integration
 def test_databases(admin, conn_string, bulkload_content):
-    assert len(admin.databases()) == 0
+    current_databases = len(admin.databases())
 
     # create database
     db = admin.new_database("db", {"search.enabled": True, "spatial.enabled": True})
 
-    assert len(admin.databases()) == 1
+    assert len(admin.databases()) == current_databases + 1
     assert db.name == "db"
     assert db.get_options("search.enabled", "spatial.enabled") == {
         "search.enabled": True,
@@ -321,7 +331,7 @@ def test_databases(admin, conn_string, bulkload_content):
     db.drop()
     bl.drop()
 
-    assert len(admin.databases()) == 0
+    assert len(admin.databases()) == current_databases
 
 
 def test_users(admin, conn_string):
diff --git a/test/test_connection.py b/test/test_connection.py
index 7cd3925..dad2e96 100644
--- a/test/test_connection.py
+++ b/test/test_connection.py
@@ -414,7 +414,7 @@ def test_invalid_request_session(conn_string):
     session = "im an invalid session"
     with pytest.raises(
         Exception,
-        match=f"type\(session\) = {type(session)} must be a valid requests\.Session object.",
+        match=f"type\\(session\\) = {type(session)} must be a valid requests\\.Session object.",
     ):
         connection.Connection("somevaliddb", **conn_string, session=session)
 
diff --git a/utils/run_tests.sh b/utils/run_tests.sh
index 07876f4..40adc35 100755
--- a/utils/run_tests.sh
+++ b/utils/run_tests.sh
@@ -2,7 +2,8 @@
 
 source ./utils/wait.sh
 
+echo "Commenting and sleeping instead, to check whether there is a bug with cluster status"
 wait_for_start ${STARDOG_LB} 5820
-# Disabling because of https://github.com/stardog-union/pystardog/issues/111
-# wait_for_standby_node ${STARDOG_HOSTNAME_STANDBY} 5820
+wait_for_standby_node ${STARDOG_HOSTNAME_STANDBY} 5820
 pytest --endpoint http://${STARDOG_LB}:5820 -s -k 'not test_single_node'
+