Repository: irods/irods_capability_storage_tiering
Pull request #348: [#290] Escape data object paths for GenQuery (main)
Author: login: alanking
Created: 2025-04-02T20:51:06Z | Merged: 2025-04-02T22:57:24Z

In service of #290 

This fixes the one known failing test by escaping logical paths before using them with internal GenQueries. 

Also fixed some include ordering issues (and ran clang-format). I kept that in a separate commit to make the salient change easier to see.

Running full test suite now.


Diff:
diff --git a/include/irods/private/storage_tiering/data_verification_utilities.hpp b/include/irods/private/storage_tiering/data_verification_utilities.hpp
index 6751146..fe84cc4 100644
--- a/include/irods/private/storage_tiering/data_verification_utilities.hpp
+++ b/include/irods/private/storage_tiering/data_verification_utilities.hpp
@@ -2,8 +2,8 @@
 #define IRODS_CAPABILITY_STORAGE_TIERING_DATA_VERIFICATION_UTILITIES_HPP
 
 #include <irods/rcConnect.h>
+
 #include <string>
-#include "irods/private/storage_tiering/storage_tiering.hpp"
 
 namespace irods {
     bool verify_replica_for_destination_resource(
diff --git a/include/irods/private/storage_tiering/storage_tiering.hpp b/include/irods/private/storage_tiering/storage_tiering.hpp
index ca41831..3ea8b7e 100644
--- a/include/irods/private/storage_tiering/storage_tiering.hpp
+++ b/include/irods/private/storage_tiering/storage_tiering.hpp
@@ -1,12 +1,17 @@
 #ifndef IRODS_CAPABILITY_STORAGE_TIERING_HPP
 #define IRODS_CAPABILITY_STORAGE_TIERING_HPP
 
-#include <list>
+#include "irods/private/storage_tiering/configuration.hpp"
+
+#include <irods/rcMisc.h>
+
 #include <boost/any.hpp>
+
+#include <list>
 #include <string>
 
-#include <irods/rcMisc.h>
-#include "irods/private/storage_tiering/configuration.hpp"
+struct RcComm;
+struct RuleExecInfo;
 
 namespace irods {
     using resource_index_map = std::map<std::string, std::string>;
@@ -23,10 +28,7 @@ namespace irods {
             static const std::string data_movement;
         };
 
-        storage_tiering(
-            rcComm_t*          _comm,
-            ruleExecInfo_t*    _rei,
-            const std::string& _instance_name);
+        storage_tiering(RcComm* _comm, RuleExecInfo* _rei, const std::string& _instance_name);
 
         void apply_policy_for_tier_group(
             const std::string& _group);
@@ -47,128 +49,87 @@ namespace irods {
             const std::string& _destination_resource);
 
         private:
+          void set_migration_metadata_flag_for_object(RcComm* _comm, const std::string& _object_path);
 
-        void set_migration_metadata_flag_for_object(
-            rcComm_t*          _comm,
-            const std::string& _object_path);
+          void unset_migration_metadata_flag_for_object(RcComm* _comm, const std::string& _object_path);
 
-        void unset_migration_metadata_flag_for_object(
-            rcComm_t*          _comm,
-            const std::string& _object_path);
+          bool object_has_migration_metadata_flag(RcComm* _comm, const std::string& _object_path);
 
-        bool object_has_migration_metadata_flag(
-            rcComm_t*          _comm,
-            const std::string& _object_path);
-
-        bool skip_object_in_lower_tier(
-            rcComm_t*          _comm,
-            const std::string& _object_path,
-            const std::string& _partial_list);
+          bool skip_object_in_lower_tier(RcComm* _comm,
+                                         const std::string& _object_path,
+                                         const std::string& _partial_list);
 
-        std::string make_partial_list(
-            resource_index_map::iterator _itr,
-            resource_index_map::iterator _end);
+          std::string make_partial_list(resource_index_map::iterator _itr, resource_index_map::iterator _end);
 
-        void update_access_time_for_data_object(
-            const std::string& _object_path);
+          void update_access_time_for_data_object(const std::string& _object_path);
 
-        std::string get_metadata_for_data_object(
-            rcComm_t*          _comm,
-            const std::string& _meta_attr_name,
-            const std::string& _object_path);
+          std::string get_metadata_for_data_object(RcComm* _comm,
+                                                   const std::string& _meta_attr_name,
+                                                   const std::string& _object_path);
 
-        std::string get_metadata_for_resource(
-            rcComm_t*          _comm,
-            const std::string& _meta_attr_name,
-            const std::string& _resource_name);
+          std::string get_metadata_for_resource(RcComm* _comm,
+                                                const std::string& _meta_attr_name,
+                                                const std::string& _resource_name);
 
-        typedef std::vector<std::pair<std::string, std::string>> metadata_results;
-        void get_metadata_for_resource(
-            rcComm_t*          _comm,
-            const std::string& _meta_attr_name,
-            const std::string& _resource_name,
-            metadata_results&  _results);
-
-        resource_index_map
-        get_tier_group_resource_ids_and_indices(
-            rcComm_t*          _comm,
-            const std::string& _group_name);
+          typedef std::vector<std::pair<std::string, std::string>> metadata_results;
+          void get_metadata_for_resource(RcComm* _comm,
+                                         const std::string& _meta_attr_name,
+                                         const std::string& _resource_name,
+                                         metadata_results& _results);
 
-        std::string get_leaf_resources_string(
-            const std::string& _resource_name);
+          resource_index_map get_tier_group_resource_ids_and_indices(RcComm* _comm, const std::string& _group_name);
 
-        bool get_preserve_replicas_for_resc(
-            rcComm_t*          _comm,
-            const std::string& _source_resource);
+          std::string get_leaf_resources_string(const std::string& _resource_name);
 
-        std::string get_verification_for_resc(
-            rcComm_t*          _comm,
-            const std::string& _resource_name);
+          bool get_preserve_replicas_for_resc(RcComm* _comm, const std::string& _source_resource);
 
-        std::string get_restage_tier_resource_name(
-            rcComm_t*          _comm,
-            const std::string& _resource_name);
+          std::string get_verification_for_resc(RcComm* _comm, const std::string& _resource_name);
 
-        auto get_group_tier_for_resource(RcComm* _comm,
-                                         const std::string& _resource_name,
-                                         const std::string& _group_name) -> int;
+          std::string get_restage_tier_resource_name(RcComm* _comm, const std::string& _resource_name);
 
-        std::string get_data_movement_parameters_for_resource(
-            rcComm_t*          _comm,
-            const std::string& _resource_name);
-
-        std::string get_replica_number_for_resource(
-            rcComm_t*          _comm,
-            const std::string& _object_path,
-            const std::string& _resource_name);
+          auto get_group_tier_for_resource(RcComm* _comm,
+                                           const std::string& _resource_name,
+                                           const std::string& _group_name) -> int;
 
-        std::string get_group_name_for_object(
-            rcComm_t*          _comm,
-            const std::string& _attribute_name,
-            const std::string& _object_path);
+          std::string get_data_movement_parameters_for_resource(RcComm* _comm, const std::string& _resource_name);
 
-        resource_index_map
-        get_resource_map_for_group(
-            rcComm_t*          _comm,
-            const std::string& _group);
+          std::string get_replica_number_for_resource(RcComm* _comm,
+                                                      const std::string& _object_path,
+                                                      const std::string& _resource_name);
 
-        std::string get_tier_time_for_resc(
-            rcComm_t*          _comm,
-            const std::string& _resource_name);
+          std::string get_group_name_for_object(RcComm* _comm,
+                                                const std::string& _attribute_name,
+                                                const std::string& _object_path);
 
-        metadata_results get_violating_queries_for_resource(
-            rcComm_t*          _comm,
-            const std::string& _resource_name);
+          resource_index_map get_resource_map_for_group(RcComm* _comm, const std::string& _group);
 
-        uint32_t get_object_limit_for_resource(
-            rcComm_t*          _comm,
-            const std::string& _resource_name);
+          std::string get_tier_time_for_resc(RcComm* _comm, const std::string& _resource_name);
 
-        void queue_data_movement(
-            rcComm_t*          _comm,
-            const std::string& _plugin_instance_name,
-            const std::string& _group_name,
-            const std::string& _object_path,
-            const std::string& _source_replica_number,
-            const std::string& _source_resource,
-            const std::string& _destination_resource,
-            const std::string& _verification_type,
-            const bool         _preserve_replicas,
-            const std::string& _data_movement_params);
+          metadata_results get_violating_queries_for_resource(RcComm* _comm, const std::string& _resource_name);
 
-        void migrate_violating_data_objects(
-            rcComm_t*           _comm,
-            const std::string&  _group_name,
-            const std::string&  _partial_list,
-            const std::string&  _source_resource,
-            const std::string&  _destination_resource);
+          uint32_t get_object_limit_for_resource(RcComm* _comm, const std::string& _resource_name);
 
-        // Attributes
-        ruleExecInfo_t*               rei_;
-        rcComm_t*                     comm_;
-        storage_tiering_configuration config_;
+          void queue_data_movement(RcComm* _comm,
+                                   const std::string& _plugin_instance_name,
+                                   const std::string& _group_name,
+                                   const std::string& _object_path,
+                                   const std::string& _source_replica_number,
+                                   const std::string& _source_resource,
+                                   const std::string& _destination_resource,
+                                   const std::string& _verification_type,
+                                   const bool _preserve_replicas,
+                                   const std::string& _data_movement_params);
 
+          void migrate_violating_data_objects(RcComm* _comm,
+                                              const std::string& _group_name,
+                                              const std::string& _partial_list,
+                                              const std::string& _source_resource,
+                                              const std::string& _destination_resource);
 
+          // Attributes
+          RuleExecInfo* rei_;
+          RcComm* comm_;
+          storage_tiering_configuration config_;
     }; // class storage_tiering
 }; // namespace irods
 
diff --git a/src/data_verification_utilities.cpp b/src/data_verification_utilities.cpp
index 1ed5ba1..d1ca437 100644
--- a/src/data_verification_utilities.cpp
+++ b/src/data_verification_utilities.cpp
@@ -1,14 +1,16 @@
-
+// TODO(#302): Remove this - we are in the server.
 #undef RODS_SERVER
 
-#include <irods/irods_query.hpp>
-#include <irods/irods_resource_manager.hpp>
-#include <irods/physPath.hpp>
 #include "irods/private/storage_tiering/data_verification_utilities.hpp"
+
 #include "irods/private/storage_tiering/configuration.hpp"
 
 #include <irods/dataObjChksum.h>
+#include <irods/escape_utilities.hpp>
 #include <irods/fileStat.h>
+#include <irods/irods_query.hpp>
+#include <irods/irods_resource_manager.hpp>
+#include <irods/physPath.hpp>
 
 #include <boost/lexical_cast.hpp>
 
@@ -95,9 +97,9 @@ namespace {
         namespace bfs = boost::filesystem;
 
         try {
-            bfs::path p(_object_path);
+            bfs::path p{irods::single_quotes_to_hex(_object_path)};
             _collection_name = p.parent_path().string();
-            _object_name     = p.filename().string();
+            _object_name = p.filename().string();
         }
         catch(const bfs::filesystem_error& _e) {
             THROW(SYS_INVALID_FILE_PATH, _e.what());
diff --git a/src/main.cpp b/src/main.cpp
index 14d2043..8764c3c 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -1,29 +1,27 @@
-// =-=-=-=-=-=-=-
-// irods includes
-#include <irods/irods_re_plugin.hpp>
+#include "irods/private/storage_tiering/data_verification_utilities.hpp"
 #include "irods/private/storage_tiering/storage_tiering.hpp"
-#include <irods/irods_re_ruleexistshelper.hpp>
 #include "irods/private/storage_tiering/utilities.hpp"
-#include <irods/irods_resource_backport.hpp>
 
+#include <irods/apiNumber.h>
 #include <irods/client_connection.hpp>
-#include <irods/modAVUMetadata.h>
-#include <irods/openCollection.h>
-#include <irods/readCollection.h>
 #include <irods/closeCollection.h>
-#include <irods/irods_logger.hpp>
-#include <irods/irods_virtual_path.hpp>
 #include <irods/dataObjRepl.h>
 #include <irods/dataObjTrim.h>
-#include <irods/physPath.hpp>
-#include <irods/apiNumber.h>
-#include "irods/private/storage_tiering/data_verification_utilities.hpp"
-#include <irods/irods_server_api_call.hpp>
-
+#include <irods/escape_utilities.hpp>
 #include <irods/irods_at_scope_exit.hpp>
+#include <irods/irods_logger.hpp>
 #include <irods/irods_query.hpp>
+#include <irods/irods_re_plugin.hpp>
+#include <irods/irods_re_ruleexistshelper.hpp>
+#include <irods/irods_resource_backport.hpp>
 #include <irods/irods_rs_comm_query.hpp>
+#include <irods/irods_server_api_call.hpp>
+#include <irods/irods_virtual_path.hpp>
+#include <irods/modAVUMetadata.h>
+#include <irods/openCollection.h>
+#include <irods/physPath.hpp>
 #include <irods/rcMisc.h>
+#include <irods/readCollection.h>
 
 #define IRODS_FILESYSTEM_ENABLE_SERVER_SIDE_API
 #include <irods/filesystem.hpp>
@@ -100,7 +98,7 @@ namespace {
     {
         namespace fs = irods::experimental::filesystem;
 
-        const auto object_path = fs::path{_object_path};
+        const auto object_path = fs::path{irods::single_quotes_to_hex(_object_path)};
 
         const auto query_string = fmt::format("select DATA_ID where DATA_NAME = '{0}' and COLL_NAME = '{1}' and "
                                               "DATA_RESC_HIER like '{2};%' || = '{2}' and DATA_REPL_STATUS = '1'",
diff --git a/src/storage_tiering.cpp b/src/storage_tiering.cpp
index 88e8f65..04f43de 100644
--- a/src/storage_tiering.cpp
+++ b/src/storage_tiering.cpp
@@ -1,27 +1,28 @@
-
+// TODO(#302): Remove this - we are in the server.
 #undef RODS_SERVER
 
-#include <irods/irods_server_properties.hpp>
-#include <irods/irods_re_plugin.hpp>
 #include "irods/private/storage_tiering/storage_tiering.hpp"
+
 #include "irods/private/storage_tiering/utilities.hpp"
-#include <irods/irods_query.hpp>
-#include <irods/irods_virtual_path.hpp>
-#include <irods/irods_hierarchy_parser.hpp>
-#include <irods/irods_resource_manager.hpp>
-#include <irods/irods_resource_backport.hpp>
-#include <irods/query_processor.hpp>
 
 #include <irods/client_connection.hpp>
+#include <irods/escape_utilities.hpp>
+#include <irods/execMyRule.h>
+#include <irods/irods_hierarchy_parser.hpp>
+#include <irods/irods_logger.hpp>
+#include <irods/irods_query.hpp>
+#include <irods/irods_re_plugin.hpp>
+#include <irods/irods_resource_backport.hpp>
+#include <irods/irods_resource_manager.hpp>
+#include <irods/irods_server_properties.hpp>
+#include <irods/irods_virtual_path.hpp>
 #include <irods/modAVUMetadata.h>
+#include <irods/objInfo.h>
+#include <irods/query_processor.hpp>
+#include <irods/rsCloseCollection.hpp>
 #include <irods/rsExecMyRule.hpp>
-#include <irods/execMyRule.h>
 #include <irods/rsOpenCollection.hpp>
 #include <irods/rsReadCollection.hpp>
-#include <irods/rsCloseCollection.hpp>
-#include <irods/irods_logger.hpp>
-
-#include "irods/private/storage_tiering/data_verification_utilities.hpp"
 
 #include <boost/any.hpp>
 #include <boost/regex.hpp>
@@ -75,7 +76,7 @@ namespace irods {
         rcComm_t*          _comm,
         const std::string& _meta_attr_name,
         const std::string& _object_path ) {
-        boost::filesystem::path p{_object_path};
+        boost::filesystem::path p{irods::single_quotes_to_hex(_object_path)};
         std::string coll_name = p.parent_path().string();
         std::string data_name = p.filename().string();
 
@@ -548,7 +549,7 @@ namespace irods {
         rcComm_t*          _comm,
         const std::string& _object_path,
         const std::string& _partial_list) {
-        boost::filesystem::path p{_object_path};
+        boost::filesystem::path p{irods::single_quotes_to_hex(_object_path)};
         std::string coll_name = p.parent_path().string();
         std::string data_name = p.filename().string();
         const auto qstr =
@@ -808,7 +809,7 @@ namespace irods {
         rcComm_t*          _comm,
         const std::string& _object_path,
         const std::string& _resource_name) {
-        boost::filesystem::path p{_object_path};
+        boost::filesystem::path p{irods::single_quotes_to_hex(_object_path)};
         std::string coll_name = p.parent_path().string();
         std::string data_name = p.filename().string();
 
@@ -835,7 +836,7 @@ namespace irods {
         rcComm_t*          _comm,
         const std::string& _attribute_name,
         const std::string& _object_path) {
-        boost::filesystem::path p{_object_path};
+        boost::filesystem::path p{irods::single_quotes_to_hex(_object_path)};
         std::string data_name = p.filename().string();
         std::string coll_name = p.parent_path().string();
 
@@ -1018,7 +1019,7 @@ namespace irods {
     bool storage_tiering::object_has_migration_metadata_flag(
         rcComm_t*          _comm,
         const std::string& _object_path) {
-        boost::filesystem::path p{_object_path};
+        boost::filesystem::path p{irods::single_quotes_to_hex(_object_path)};
         std::string coll_name = p.parent_path().string();
         std::string data_name = p.filename().string();
 


Reviews:
user: login: korydraughn
body: Once the review comment is handled, go ahead and squash it.
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-02T21:19:06Z
user: login: alanking
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T21:58:33Z
user: login: korydraughn
body: Nice. Squash to taste.
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-02T22:27:11Z
user: login: korydraughn
body: Pound it.
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-02T22:53:00Z

Review comments:
diff_hunk: @@ -1,12 +1,16 @@
 #ifndef IRODS_CAPABILITY_STORAGE_TIERING_HPP
 #define IRODS_CAPABILITY_STORAGE_TIERING_HPP
 
-#include <list>
+#include "irods/private/storage_tiering/configuration.hpp"
+#include <irods/rcMisc.h>
path: include/irods/private/storage_tiering/storage_tiering.hpp
user: login: korydraughn
body: Let's add an empty line between these headers so it's clear that one is provided by the tiering implementation and the other is from an external source.
created_at: 2025-04-02T21:18:02Z
author_association: COLLABORATOR
diff_hunk: @@ -1,12 +1,16 @@
 #ifndef IRODS_CAPABILITY_STORAGE_TIERING_HPP
 #define IRODS_CAPABILITY_STORAGE_TIERING_HPP
 
-#include <list>
+#include "irods/private/storage_tiering/configuration.hpp"
+#include <irods/rcMisc.h>
path: include/irods/private/storage_tiering/storage_tiering.hpp
user: login: alanking
body: Done
created_at: 2025-04-02T21:58:32Z
author_association: CONTRIBUTOR

PR comments:
user: login: alanking
created_at: 2025-04-02T22:00:14Z
body: I noticed one spot where we can escape the full path once rather than the parent collection and object name separately. That's the commit against which tests were run, and all the tests passed.
author_association: CONTRIBUTOR
user: login: alanking
created_at: 2025-04-02T22:33:46Z
body: Squashed
author_association: CONTRIBUTOR
user: login: alanking
created_at: 2025-04-02T22:57:20Z
body: #'d, mergin
author_association: CONTRIBUTOR