Repository: skjerns/mat7.3
Pull request #62: correctly load in 2D char arrays
Author: login: sjvenditto
Created: 2025-05-21T18:10:57Z | Merged: 2025-05-26T13:44:08Z

This is addressing Issue #61.  I've made a small fix when loading in char arrays to handle 2D shapes. Currently, I have it return a list of strings that matches the original format in Matlab. I also added a short test for this, which loads in and checks the char array that I noted in the issue.

Let me know if there's anything else that should be fixed or changed with how I implemented this!


Diff:
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
index 1ed6958..ee12f19 100644
--- a/.github/workflows/run-tests.yml
+++ b/.github/workflows/run-tests.yml
@@ -15,7 +15,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11']
         os: [ubuntu-latest, macos-latest, windows-latest]
         exclude:
           - os: macos-latest
diff --git a/mat73/core.py b/mat73/core.py
index 3db30a9..f932d28 100644
--- a/mat73/core.py
+++ b/mat73/core.py
@@ -124,7 +124,7 @@ def unpack_mat(self, hdf5, depth=0, MATLAB_class=None, force=False):
                 else:
                     data = []
                     row_ind = []
-                
+
                 col_ind = hdf5['jc']
                 n_rows = hdf5.attrs['MATLAB_sparse']
                 n_cols = len(col_ind) - 1
@@ -264,9 +264,31 @@ def convert_mat(self, dataset, depth, MATLAB_class=None):
                 return None
 
         elif mtype=='char':
-            string_array = np.ravel(dataset)
-            string_array = ''.join([chr(x) for x in string_array])
-            return string_array
+            codes = np.asarray(dataset, dtype=np.uint16)
+
+            # object dtype → keeps '\x00'
+            # see https://github.com/numpy/numpy/issues/28964
+            to_char = np.vectorize(chr, otypes=[object])
+            arr = to_char(codes)
+
+            char_axis = 0 if arr.ndim < 3 else -2
+            char_arr = np.apply_along_axis(lambda x: ''.join(x), axis=char_axis, arr=arr)
+
+            string_list = char_arr.tolist()
+
+            if arr.ndim==2 and arr.shape[1]==1:
+                string_list = string_list[0]
+
+            if arr.ndim>2:
+                # print warning to be sure. I haven't encountered any char
+                # arrays with ndim>2 in the wild yet so can't be sure that
+                # they are actually the way I synthesized them
+                logging.warning(f"Loading char array '{dataset.name}' with {arr.ndim} dimensions "
+                                f"might be wrong stacked (i.e. dimensions scrambled). "
+                                f"please check variable is correct and report errors "
+                                f"on github.com/skjerns/mat7.3")
+
+            return string_list
 
         elif mtype=='bool':
             return bool(dataset)
@@ -306,13 +328,13 @@ def convert_mat(self, dataset, depth, MATLAB_class=None):
 
 def squeeze(arr):
     """Vectors are saved as 2D matrices in MATLAB, however in numpy
-    there is no distinction between a column and a row vector. 
+    there is no distinction between a column and a row vector.
     Therefore, remove superfluous dimensions if the array is 2D and
     one of the dimensions is singular"""
     if arr.ndim==2 and 1 in arr.shape:
         return arr.reshape([x for x in arr.shape if x>1])
     return arr
-    
+
 
 def loadmat(file, use_attrdict=False, only_include=None, verbose=True):
     """
@@ -359,8 +381,5 @@ def savemat(filename, verbose=True):
 
 if __name__=='__main__':
     # for testing / debugging
-    d = loadmat('../tests/testfile11.mat', only_include='foo')
-
-
-    # file = '../tests/testfile8.mat'
-    # data = loadmat(file)
+    d = loadmat('../tests/testfile16.mat')
+    print(d)
diff --git a/tests/create_mat.m b/tests/create_mat.m
index ab8cc23..f2cb73b 100644
--- a/tests/create_mat.m
+++ b/tests/create_mat.m
@@ -65,3 +65,20 @@
 clear all
 
 
+%% file to test 2D char arrays
+char_arr_1d = ['abcd']
+char_arr_2d = [ ...
+    'PSTH tensor for image sequences (averaged across frames):'; ...
+    'dimension 1: 2 scales (zoom1x, zoom2x)                   '; ...
+    'dimension 2: 3 category (natural, synthetic, contrast)   '; ...
+    'dimension 3: 10 movies                                   '; ...
+    'dimension 4: sorted units                                '; ...
+    'dimension 5: PSTH time bins                              '];
+
+char_arr_3d = cat(3, ...
+    ['abcd'; 'defg'], ...  % First "page"
+    ['ghij'; 'jklm'], ...  % Second "page"
+    ['mnöp'; 'pqrs'])     % Third "page"
+
+save('testfile16.mat','char_arr_1d', 'char_arr_2d', 'char_arr_3d', '-v7.3')
+clear all
diff --git a/tests/test_mat73.py b/tests/test_mat73.py
index 5302102..2bada4a 100644
--- a/tests/test_mat73.py
+++ b/tests/test_mat73.py
@@ -15,7 +15,7 @@
     version = pkg_resources.get_distribution('mat73').version
 except:
     version = '0.00'
-    
+
 try:
     repo = Repository('.')
     head = repo.head
@@ -35,7 +35,7 @@ class Testing(unittest.TestCase):
 
     def setUp(self):
         """make links to test files and make sure they are present"""
-        for i in range(1, 16):
+        for i in range(1, 17):
             file = 'testfile{}.mat'.format(i)
             if not os.path.exists(file):
                 file = os.path.join('./tests', file)
@@ -390,7 +390,7 @@ def test_load_specific_vars(self):
         elapsed2 = time.time()-start
         assert elapsed2<elapsed1, 'loading specific var was not faster'
 
-    def test_file10_nullchars(self):
+    def test_file8_nullchars(self):
         """test if null chars are retained in char arrays"""
         data = mat73.loadmat(self.testfile8)
         self.assertEqual(len(data['char_array']), 7, 'not all elements loaded')
@@ -503,12 +503,27 @@ def test_file15_strip(self):
                     'x_1_1_10_1_1': (1, 1, 10),
                     'x_10_1_1_10': (10, 1, 1, 10),
                     }
-        
-        
+
+
         for var, shape in expected.items():
             self.assertEqual(data[var].shape, shape)
             self.assertEqual(data[var].ndim, len(shape))
 
+    def test_file16_2d_char_array(self):
+        """Test loading of 2D char array"""
+        # the matlab array has shape (6 X 57)
+
+        data = mat73.loadmat(self.testfile16)
+
+        expected = ['PSTH tensor for image sequences (averaged across frames):',
+                    'dimension 1: 2 scales (zoom1x, zoom2x)                   ',
+                    'dimension 2: 3 category (natural, synthetic, contrast)   ',
+                    'dimension 3: 10 movies                                   ',
+                    'dimension 4: sorted units                                ',
+                    'dimension 5: PSTH time bins                              ']
+        self.assertEqual(data['char_arr_1d'], 'abcd')
+        self.assertEqual(data['char_arr_2d'], expected)
+        self.assertEqual(data['char_arr_3d'], [['abcd', 'defg'], ['ghij', 'jklm'], ['mnöp', 'pqrs']])
 
 if __name__ == '__main__':
 
diff --git a/tests/testfile16.mat b/tests/testfile16.mat
new file mode 100644
index 0000000..d4c52ff
Binary files /dev/null and b/tests/testfile16.mat differ


Reviews:
user: login: gemini-code-assist[bot]
body: Hello @sjvenditto, I'm Gemini Code Assist[^1]! I'm currently reviewing this pull request and will post my feedback shortly. In the meantime, here's a summary to help you and other reviewers quickly get up to speed!

## Summary of Changes
Hello! Gemini or gemini-code-assist here, providing a summary of this pull request. This PR addresses Issue #61 by implementing a fix for loading 2D character arrays from MATLAB v7.3 `.mat` files. Previously, these arrays were incorrectly flattened into a single string. The changes modify the `mat73.core.convert_mat` function to detect 2D character arrays and load them as a list of strings, preserving the row structure as seen in MATLAB. A new MATLAB script is added to generate a test file containing a 2D character array, and a corresponding Python test case is included to verify the correct loading behavior.

### Highlights
* **Bug Fix**: Correctly loads 2D character arrays from MATLAB .mat files, resolving Issue #61.
* **Data Structure**: 2D character arrays are now loaded as a list of strings, where each string represents a row from the original MATLAB array.
* **Testing**: Adds a new MATLAB script (`create_mat.m`) to generate a test file with a 2D character array and a new Python test case (`test_mat73.py`) to validate the fix.

### Changelog
* **mat73/core.py**
  * Modified the `convert_mat` function to check for 2D character arrays (`dataset.shape[1] > 1`).
  * Implemented logic to extract rows from the flattened array based on the original shape and MATLAB's column-major order.
  * Returns a list of strings for 2D character arrays, falling back to a single string for 1D arrays.
* **tests/create_mat.m**
  * Added a new section to create `testfile16.mat` containing a sample 2D character array (`char_arr_2d`).
* **tests/test_mat73.py**
  * Updated the `setUp` method to include `testfile16.mat` in the list of test files.
  * Added a new test method `test_file16_2d_char_array` to load `testfile16.mat` and assert the correct loading of the 2D character array as a list of strings.


<details>

<summary><b>Using Gemini Code Assist</b></summary>
<br>

The full guide for Gemini Code Assist can be found on our [documentation page](https://developers.google.com/gemini-code-assist/docs/review-github-code), here are some quick tips.

<b>Invoking Gemini</b>

You can request assistance from Gemini at any point in your pull request via creating an issue comment (i.e. comment on the pull request page) using either `/gemini <command>` or `@gemini-code-assist <command>`. Below is a summary of the supported commands.

Feature | Command | Description
--- | --- | ---
Code Review | `/gemini review` | Performs a code review for the current pull request in its current state.
Pull Request Summary | `/gemini summary` | Provides a summary of the current pull request in its current state.
Comment | @gemini-code-assist | Responds in comments when explicitly tagged, both in issue comments and review comments.
Help | `/gemini help` | Displays a list of available commands.

<b>Customization</b>

To customize Gemini Code Assist for GitHub experience, repository maintainers can create a configuration file and/or provide a custom code review style guide (such as PEP-8 for Python) by creating and adding files to a `.gemini/` folder in the base of the repository. Detailed instructions can be found [here](https://developers.google.com/gemini-code-assist/docs/customize-gemini-behavior-github).

<b>Limitations & Feedback</b>

Gemini Code Assist is currently in preview and may make mistakes. Please leave feedback on any instances where its feedback is incorrect or counter productive. You can react with :thumbsup: and :thumbsdown: on @gemini-code-assist comments to provide feedback.

</details>

<b>You can also get AI-powered code generation, chat, as well as code reviews directly in the IDE at no cost with the [Gemini Code Assist IDE Extension](https://cloud.google.com/products/gemini/code-assist).</b>


[^1]: Review the [Privacy Notices](https://policies.google.com/privacy), [Generative AI Prohibited Use Policy](https://policies.google.com/terms/generative-ai/use-policy), [Terms of Service](https://policies.google.com/terms), and learn how to configure Gemini Code Assist in GitHub [here](https://developers.google.com/gemini-code-assist/docs/review-github-code). Gemini can make mistakes, so double check it and [use code with caution](https://support.google.com/legal/answer/13505487).

---

> MATLAB data flows,
> Python reads the rows it knows,
> Strings in a list.


state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T18:11:15Z
user: login: gemini-code-assist[bot]
body: ## Code Review
This pull request introduces a fix for loading 2D char arrays from MATLAB files, ensuring they are represented as a list of strings in Python. The addition of a dedicated test case (`testfile16.mat` and corresponding Python test) is excellent for verifying this functionality.

However, there's a potential `IndexError` in `mat73/core.py` where accessing `dataset.shape[1]` can lead to an `IndexError` if the `dataset` is a 1D char array. This needs to be fixed to ensure robust handling of all char array types.

Once the critical issue is addressed, this PR will be a good improvement to the library.

### Summary of Findings
* **Potential IndexError for 1D char arrays**: The code in `mat73/core.py` accesses `dataset.shape[1]` without first checking if `dataset` is 1D. This will cause an `IndexError` if a 1D char array is processed, as 1D arrays do not have a `shape[1]`.

### Merge Readiness
The pull request addresses an important issue with loading 2D char arrays and includes a good test case. However, due to the potential `IndexError` bug identified in `mat73/core.py` when handling 1D char arrays, I recommend that these changes **not be merged** until this issue is resolved. My code suggestion aims to fix this. I am unable to approve pull requests myself; please ensure the fix is reviewed and approved by a maintainer before merging.
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-21T18:13:55Z
user: login: skjerns
state: COMMENTED
author_association: OWNER
submitted_at: 2025-05-22T08:14:10Z
user: login: sjvenditto
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-22T14:30:47Z

Review comments:
diff_hunk: @@ -265,7 +265,11 @@ def convert_mat(self, dataset, depth, MATLAB_class=None):
 
         elif mtype=='char':
             string_array = np.ravel(dataset)
-            string_array = ''.join([chr(x) for x in string_array])
+            if dataset.shape[1] > 1:
+                # MATLAB uses column-major order, so we need to transpose
+                string_array = [''.join([chr(x) for x in string_array][n::dataset.shape[1]]) for n in range(dataset.shape[1])]
path: mat73/core.py
user: login: gemini-code-assist[bot]
body: ![high](https://img.shields.io/badge/priority-high-orange)

If `dataset` is a 1D char array (e.g., `shape` is `(L,)`), accessing `dataset.shape[1]` will raise an `IndexError` because 1D arrays don't have a second dimension. MATLAB can produce 1D char arrays (which are essentially strings). The new logic needs to ensure 1D arrays are still processed correctly, likely falling into the `else` path or a similar path that produces a single string.

Could you refactor this block to first check `dataset.ndim` before attempting to access `dataset.shape[1]`? This would make the handling of 1D and 2D char arrays more robust.

```suggestion
            if dataset.ndim == 2 and dataset.shape[1] > 1:
                # MATLAB uses column-major order, so we need to transpose
                string_array = [''.join([chr(x) for x in string_array][n::dataset.shape[1]]) for n in range(dataset.shape[1])]
```
created_at: 2025-05-21T18:13:55Z
author_association: CONTRIBUTOR
diff_hunk: @@ -265,7 +265,11 @@ def convert_mat(self, dataset, depth, MATLAB_class=None):
 
         elif mtype=='char':
             string_array = np.ravel(dataset)
-            string_array = ''.join([chr(x) for x in string_array])
+            if dataset.ndim == 2 and dataset.shape[1] > 1:
path: mat73/core.py
user: login: skjerns
body: Not too familiar with MATLAB, but can a MATLAB char array every be `ndim>2`?
created_at: 2025-05-22T08:14:10Z
author_association: OWNER
diff_hunk: @@ -265,7 +265,11 @@ def convert_mat(self, dataset, depth, MATLAB_class=None):
 
         elif mtype=='char':
             string_array = np.ravel(dataset)
-            string_array = ''.join([chr(x) for x in string_array])
+            if dataset.ndim == 2 and dataset.shape[1] > 1:
path: mat73/core.py
user: login: sjvenditto
body: Yes, they're rare since all strings have to be the same length, but there's a dataset I've been working with where someone saved their data in this format by padding the lines with spaces (it was the char array that I put in the issue). I loaded it in Matlab to verify how it was supposed to be formatted.

Also, from the same file, I noticed that even 1D char arrays were parsed with 2 dimensions with shape (n, 1). Don't know how true this is in general, which is why I accepted the change for checking `ndim` first
created_at: 2025-05-22T14:30:47Z
author_association: CONTRIBUTOR

PR comments:
user: login: skjerns
created_at: 2025-05-23T16:31:29Z
body: I've tried the MATLAB file you provided, but with my MATLAB R2024b the array is somehow not 2d, but saved as a 1D with `shape=(342, 1)`, very strange! However, creating a 3d char array was possible for me. Any idea what might the the problem here?

![image](https://github.com/user-attachments/assets/6fae60f2-0102-4897-a578-45d1e18b4533)
![image](https://github.com/user-attachments/assets/f51fbed8-b913-4496-9b92-5eee0da90c84)


edit: changing `,` to `;` fixed it :) I'll push to your branch soon
author_association: OWNER
user: login: skjerns
created_at: 2025-05-23T17:14:46Z
body: @sjvenditto  I've pushed my changes to the repo - could you run the `create_mat.m` on your repo and create the new `testfile16.mat`? I somehow are not allowed to push it to your repo.

Check if the code runs on your machine

thanks for the PR and raising this issue!
author_association: OWNER
user: login: skjerns
created_at: 2025-05-25T17:46:33Z
body: Oh no, StringDType is `numpy>=2.0` only - need to think how to solve this by 

a) using different methods for string conversion
b) using a workaround for `\x00` and continue use of `np.str_`
c) making `numpy>=2` a requirement


edit: fixed it!
author_association: OWNER
user: login: skjerns
created_at: 2025-05-26T13:44:16Z
body: thanks for the addition! I've merged it to main
author_association: OWNER
user: login: sjvenditto
created_at: 2025-05-27T19:03:55Z
body: > I've tried the MATLAB file you provided, but with my MATLAB R2024b the array is somehow not 2d, but saved as a 1D with `shape=(342, 1)`, very strange! However, creating a 3d char array was possible for me. Any idea what might the the problem here?
> 
> ![image](https://private-user-images.githubusercontent.com/14980558/447099473-6fae60f2-0102-4897-a578-45d1e18b4533.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTUiLCJleHAiOjE3NDgzNzI3OTMsIm5iZiI6MTc0ODM3MjQ5MywicGF0aCI6Ii8xNDk4MDU1OC80NDcwOTk0NzMtNmZhZTYwZjItMDEwMi00ODk3LWE1NzgtNDVkMWUxOGI0NTMzLnBuZz9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUFWQ09EWUxTQTUzUFFLNFpBJTJGMjAyNTA1MjclMkZ1cy1lYXN0LTElMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjUwNTI3VDE5MDEzM1omWC1BbXotRXhwaXJlcz0zMDAmWC1BbXotU2lnbmF0dXJlPTEzNGQ2YTJhMWUwNDEyNzE3NzU3NDY0NmI5MzYwMjQxNTQ0NzJjY2UzYzcxNDA3NzJiYTFjODdmZWRjOTZiMDAmWC1BbXotU2lnbmVkSGVhZGVycz1ob3N0In0.Gv_yiYLy7MWEg3QEOliFG6FOzU5MfQv2gBwnxgtOqE8) ![image](https://private-user-images.githubusercontent.com/14980558/447099558-f51fbed8-b913-4496-9b92-5eee0da90c84.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTUiLCJleHAiOjE3NDgzNzI3OTMsIm5iZiI6MTc0ODM3MjQ5MywicGF0aCI6Ii8xNDk4MDU1OC80NDcwOTk1NTgtZjUxZmJlZDgtYjkxMy00NDk2LTliOTItNWVlZTBkYTkwYzg0LnBuZz9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUFWQ09EWUxTQTUzUFFLNFpBJTJGMjAyNTA1MjclMkZ1cy1lYXN0LTElMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjUwNTI3VDE5MDEzM1omWC1BbXotRXhwaXJlcz0zMDAmWC1BbXotU2lnbmF0dXJlPTQ0ZTk0ZTRkY2ZlNDdkYjdiZDk4ZjFkOTVhNjhiODViNzU3NGEyNDEyMjQxYTBiYWVlYjU3OTU4OGFkYTI4MzkmWC1BbXotU2lnbmVkSGVhZGVycz1ob3N0In0.JzqlUwROaJba4qX-dngpB6r5z1oxUPH3Jm27l7LmjtM)
> 
> edit: changing `,` to `;` fixed it :) I'll push to your branch soon

Sorry about this! I actually didn't run the tests on the file generated in `create_mat.m`, I ran it on a file I saved directly in MATLAB and didn't check whether `create_mat.m` made the same file
author_association: CONTRIBUTOR