Repository: single-cell-data/TileDB-SOMA (language: python)

Issue:
[c++] Apply `ArraySchemaEvolution` to specified timestamp
**Issue and/or context:** #2879 

**Changes:** If we've got the array open at a timestamp, and we need to evolve it, evolve from that same timestamp.

**Notes for Reviewer:**

Material from @nguyenv to be incorporated into unit-test material:

**Script:**

```
$ cat ts.py
import tiledb
import tiledbsoma as soma
import pyarrow as pa
import tempfile
import numpy as np
import pandas as pd
import time

with tempfile.TemporaryDirectory() as uri:
    asch = pa.schema([("foo", pa.dictionary(pa.int8(), pa.large_string()))])
    soma.DataFrame.create(uri, schema=asch, tiledb_timestamp=1).close()

    pydict = {}
    pydict["soma_joinid"] = [0, 1, 2]
    pydict["foo"] = pd.Series(['a', 'b', 'a'], dtype='category')
    rb = pa.Table.from_pydict(pydict)

    with soma.DataFrame.open(uri, "w", tiledb_timestamp=2) as sdf:
        sdf.write(rb)

    pydict = {}
    pydict["soma_joinid"] = [3, 4]
    pydict["foo"] = pd.Series(['b', 'b'], dtype='category')
    rb = pa.Table.from_pydict(pydict)

    with soma.DataFrame.open(uri, "w", tiledb_timestamp=3) as sdf:
        sdf.write(rb)

    with soma.DataFrame.open(uri) as sdf:
        table = sdf.read().concat()
        print(table)

    with soma.DataFrame.open(uri, tiledb_timestamp=1) as sdf:
        table = sdf.read().concat()
        print(table)

    with soma.DataFrame.open(uri, tiledb_timestamp=2) as sdf:
        table = sdf.read().concat()
        print(table)

    with soma.DataFrame.open(uri, tiledb_timestamp=3) as sdf:
        table = sdf.read().concat()
        print(table)
```

**Before:**

```
pyarrow.Table
soma_joinid: int64
foo: dictionary<values=string, indices=int8, ordered=0>
----
soma_joinid: [[0,1,2,3,4]]
foo: [  -- dictionary:
["a","b"]  -- indices:
[0,1,0,0,0]]
pyarrow.Table
soma_joinid: int64
foo: dictionary<values=string, indices=int8, ordered=0>
----
soma_joinid: [[]]
foo: [  -- dictionary:
[]  -- indices:
[]]
pyarrow.Table
soma_joinid: int64
foo: dictionary<values=string, indices=int8, ordered=0>
----
soma_joinid: [[0,1,2]]
foo: [  -- dictionary:
[]  -- indices:
[0,1,0]]
pyarrow.Table
soma_joinid: int64
foo: dictionary<values=string, indices=int8, ordered=0>
----
soma_joinid: [[0,1,2,3,4]]
foo: [  -- dictionary:
[]  -- indices:
[0,1,0,0,0]]
```

**After:**

```
pyarrow.Table
soma_joinid: int64
foo: dictionary<values=string, indices=int8, ordered=0>
----
soma_joinid: [[0,1,2,3,4]]
foo: [  -- dictionary:
["a","b"]  -- indices:
[0,1,0,0,0]]
pyarrow.Table
soma_joinid: int64
foo: dictionary<values=string, indices=int8, ordered=0>
----
soma_joinid: [[]]
foo: [  -- dictionary:
[]  -- indices:
[]]
pyarrow.Table
soma_joinid: int64
foo: dictionary<values=string, indices=int8, ordered=0>
----
soma_joinid: [[0,1,2]]
foo: [  -- dictionary:
["a","b"]  -- indices:
[0,1,0]]
pyarrow.Table
soma_joinid: int64
foo: dictionary<values=string, indices=int8, ordered=0>
----
soma_joinid: [[0,1,2,3,4]]
foo: [  -- dictionary:
["a","b"]  -- indices:
[0,1,0,0,0]]
```

**Difference:**

```diff
23c23
< []  -- indices:
---
> ["a","b"]  -- indices:
31c31
< []  -- indices:
---
> ["a","b"]  -- indices:
```
[c++] Apply `ArraySchemaEvolution` to specified timestamp
**Issue and/or context:** #2879 

**Changes:** If we've got the array open at a timestamp, and we need to evolve it, evolve from that same timestamp.

**Notes for Reviewer:**

Material from @nguyenv to be incorporated into unit-test material:

**Script:**

```
$ cat ts.py
import tiledb
import tiledbsoma as soma
import pyarrow as pa
import tempfile
import numpy as np
import pandas as pd
import time

with tempfile.TemporaryDirectory() as uri:
    asch = pa.schema([("foo", pa.dictionary(pa.int8(), pa.large_string()))])
    soma.DataFrame.create(uri, schema=asch, tiledb_timestamp=1).close()

    pydict = {}
    pydict["soma_joinid"] = [0, 1, 2]
    pydict["foo"] = pd.Series(['a', 'b', 'a'], dtype='category')
    rb = pa.Table.from_pydict(pydict)

    with soma.DataFrame.open(uri, "w", tiledb_timestamp=2) as sdf:
        sdf.write(rb)

    pydict = {}
    pydict["soma_joinid"] = [3, 4]
    pydict["foo"] = pd.Series(['b', 'b'], dtype='category')
    rb = pa.Table.from_pydict(pydict)

    with soma.DataFrame.open(uri, "w", tiledb_timestamp=3) as sdf:
        sdf.write(rb)

    with soma.DataFrame.open(uri) as sdf:
        table = sdf.read().concat()
        print(table)

    with soma.DataFrame.open(uri, tiledb_timestamp=1) as sdf:
        table = sdf.read().concat()
        print(table)

    with soma.DataFrame.open(uri, tiledb_timestamp=2) as sdf:
        table = sdf.read().concat()
        print(table)

    with soma.DataFrame.open(uri, tiledb_timestamp=3) as sdf:
        table = sdf.read().concat()
        print(table)
```

**Before:**

```
pyarrow.Table
soma_joinid: int64
foo: dictionary<values=string, indices=int8, ordered=0>
----
soma_joinid: [[0,1,2,3,4]]
foo: [  -- dictionary:
["a","b"]  -- indices:
[0,1,0,0,0]]
pyarrow.Table
soma_joinid: int64
foo: dictionary<values=string, indices=int8, ordered=0>
----
soma_joinid: [[]]
foo: [  -- dictionary:
[]  -- indices:
[]]
pyarrow.Table
soma_joinid: int64
foo: dictionary<values=string, indices=int8, ordered=0>
----
soma_joinid: [[0,1,2]]
foo: [  -- dictionary:
[]  -- indices:
[0,1,0]]
pyarrow.Table
soma_joinid: int64
foo: dictionary<values=string, indices=int8, ordered=0>
----
soma_joinid: [[0,1,2,3,4]]
foo: [  -- dictionary:
[]  -- indices:
[0,1,0,0,0]]
```

**After:**

```
pyarrow.Table
soma_joinid: int64
foo: dictionary<values=string, indices=int8, ordered=0>
----
soma_joinid: [[0,1,2,3,4]]
foo: [  -- dictionary:
["a","b"]  -- indices:
[0,1,0,0,0]]
pyarrow.Table
soma_joinid: int64
foo: dictionary<values=string, indices=int8, ordered=0>
----
soma_joinid: [[]]
foo: [  -- dictionary:
[]  -- indices:
[]]
pyarrow.Table
soma_joinid: int64
foo: dictionary<values=string, indices=int8, ordered=0>
----
soma_joinid: [[0,1,2]]
foo: [  -- dictionary:
["a","b"]  -- indices:
[0,1,0]]
pyarrow.Table
soma_joinid: int64
foo: dictionary<values=string, indices=int8, ordered=0>
----
soma_joinid: [[0,1,2,3,4]]
foo: [  -- dictionary:
["a","b"]  -- indices:
[0,1,0,0,0]]
```

**Difference:**

```diff
23c23
< []  -- indices:
---
> ["a","b"]  -- indices:
31c31
< []  -- indices:
---
> ["a","b"]  -- indices:
```
[c++] Corner-case bug in extend-enumeration logic
**Issue and/or context:** Fixes issue #2896.

**Changes:** Remap indices whether or not the enumeration is extended, as shown in the unit-test example added here.

**Notes for Reviewer:**
[c++] Methodize timestamped-schema-evolution factory
**Issue and/or context:** This is a line-count-reducing under-diff split out from #2785 for issue #2407; [[sc-51048]](https://app.shortcut.com/tiledb-inc/story/51048).

**Changes:**

On #2895 I made an inline code block for instantiating an `ArraySchemaEvolution` object pointed at the current array's context and open-at timestamp (if any). That was after I did a call-graph analysis that showed that this was the sole spot in `main` -- at that point in time -- where we needed to do this.

However, on #2785 we'll be making more use of schema evolution, from more than one callsite. It makes even more sense now to encapsulate this logic in a single place.

**Notes for Reviewer:**



Discussion / hints:
## [Codecov](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2895?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 89.98%. Comparing base [(`9f82dad`)](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/commit/9f82dad43da9ea3b8c09f066295900a7adb0e9f7?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) to head [(`d87cca3`)](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/commit/d87cca36bf547e1001e8e2b44442ad81ae284ffc?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data).
> Report is 4 commits behind head on main.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main    #2895      +/-   ##
==========================================
+ Coverage   89.83%   89.98%   +0.15%     
==========================================
  Files          37       37              
  Lines        3926     3926              
==========================================
+ Hits         3527     3533       +6     
+ Misses        399      393       -6     
```

| [Flag](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2895/flags?src=pr&el=flags&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) | Coverage Δ | |
|---|---|---|
| [python](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2895/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) | `89.98% <ø> (+0.15%)` | :arrow_up: |

Flags with carried forward coverage won't be shown. [Click here](https://docs.codecov.io/docs/carryforward-flags?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data#carryforward-flags-in-the-pull-request-comment) to find out more.

| [Components](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2895/components?src=pr&el=components&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) | Coverage Δ | |
|---|---|---|
| [python_api](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2895/components?src=pr&el=component&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) | `89.98% <ø> (+0.15%)` | :arrow_up: |
| [libtiledbsoma](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2895/components?src=pr&el=component&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) | `∅ <ø> (∅)` | |

</details>
Good.  That should also help for R where we documented this first,
There is something going on, since

```
["a","b"]  -- indices:
[0,1,0,0,0]]
```

should have been

```
["a","b"]  -- indices:
[0,1,0,1,1]]
```

since we wrote `'b', 'b'` at slots 3,4

```
    pydict = {}
    pydict["soma_joinid"] = [3, 4]
    pydict["foo"] = pd.Series(['b', 'b'], dtype='category')
    rb = pa.Table.from_pydict(pydict)
```
I've verified that ☝️ is a separate bug. Running this on a built checkout without this PR:

```
ubuntu@segge[prod][kerl/h2h][TileDB-SOMA]$ cat ts.py
import tiledb
import tiledbsoma as soma
import pyarrow as pa
import tempfile
import numpy as np
import pandas as pd
import time

with tempfile.TemporaryDirectory() as uri:

    asch = pa.schema([
        ("foo", pa.dictionary(pa.int8(), pa.large_string()))
    ])

    # Create at time 1
    soma.DataFrame.create(uri, schema=asch, tiledb_timestamp=1).close()

    # Write at time 2
    atbl = pa.Table.from_pydict({
        "soma_joinid": [0, 1, 2],
        "foo": pd.Series(['a', 'b', 'a'], dtype='category'),
    })
    with soma.DataFrame.open(uri, "w", tiledb_timestamp=2) as sdf:
        sdf.write(atbl)

    # Write at time 3
    atbl = pa.Table.from_pydict({
        "soma_joinid": [3, 4],
        #"foo": pd.Series(['b', 'c'], dtype='category'),
        "foo": pd.Series(['b', 'b'], dtype='category'),
    })
    with soma.DataFrame.open(uri, "w", tiledb_timestamp=3) as sdf:
        sdf.write(atbl)

    with soma.DataFrame.open(uri) as sdf:
        table = sdf.read().concat()
        print()
        print("TABLE AT NOW")
        print(table)
        print()
        print(table.to_pandas())
        print()
```

I get

```
TABLE AT NOW
pyarrow.Table
soma_joinid: int64
foo: dictionary<values=string, indices=int8, ordered=0>
----
soma_joinid: [[0,1,2,3,4]]
foo: [  -- dictionary:
["a","b"]  -- indices:
[0,1,0,0,0]]

   soma_joinid foo
0            0   a
1            1   b
2            2   a
3            3   a <------ note
4            4   a <------ note
```
> Good. That should also help for R where we documented this first,

@eddelbuettel great! If you have a ilnk for that "where" handy, please share it here -- that will be helpful.


Given #2896 and the proof above that the issue I identified is orthogonal to this PR (and pre-dates it), I think we should move forward with this in order to facilitate ongoing R-side development. And #2896 remains open.

I'll add a Python unit-test case as noted above.
> > Good. That should also help for R where we documented this first,
> > 
> @eddelbuettel great! If you have a ilnk for that "where" handy, please share it here -- that will be helpful.
> 

Never mind -- I found #2879
@johnkerl   Here is my actual test script from last week (which is not yet committed to the 'scraps' repo I use for these).  With the SOMA version I have here (a few days old) it just comes up empty (no 'strings' for the enumeration).  Worth testing on your patch.

<details>

```r
!#!/usr/bin/env Rscript                                                                                                                                                            
                                                                                                                                                                                  
uri <- "/tmp/tiledb/somadf-evolve-ts"                                                                                                                                             
if (dir.exists(uri)) unlink(uri, recursive=TRUE)                                                                                                                                  
                                                                                                                                                                                  
schema <- arrow::schema(arrow::field("soma_joinid", arrow::int64(), nullable = FALSE),                                                                                            
                        arrow::field("int", arrow::int32(), nullable = FALSE),                                                                                                    
                        arrow::field("str", arrow::dictionary(index_type = arrow::int8(), value_type = arrow::utf8(), ordered = FALSE), nullable=FALSE))                          
print(schema)                                                                                                                                                                     
ts1 <- rep(as.POSIXct(1, tz="UTC"), 2)                                                                                                                                            
sdf <- tiledbsoma::SOMADataFrameCreate(uri, schema, tiledb_timestamp=ts1[1])                                                                                                      
data <- arrow::arrow_table(soma_joinid = bit64::as.integer64(1L:5L),                                                                                                              
                           int = 101:105L,                                                                                                                                        
                           str = factor(c('a','b','b','a','b')))                                                                                                                  
cat("\n")                                                                                                                                                                         
print(tibble::as_tibble(data))                                                                                                                                                    
ts2 <- rep(as.POSIXct(2, tz="UTC"), 2)                                                                                                                                            
sdf$write(data, ts2)                                                                                                                                                              
                                                                                                                                                                                  
data <- arrow::arrow_table(soma_joinid = bit64::as.integer64(6L:10L),                                                                                                             
                           int = 106:110L,                                                                                                                                        
                           str = factor(c('c','b','c','c','b')))                                                                                                                  
cat("\n")                                                                                                                                                                         
print(tibble::as_tibble(data))                                                                                                                                                    
ts3 <- rep(as.POSIXct(3, tz="UTC"), 2)                                                                                                                                            
sdf$write(data, ts3)                                                                                                                                                              
sdf$close()                                                                                                                                                                       
                                                                                                                                                                                  
cat("\n")                                                                                                                                                                         
tiledb::tiledb_array(uri, return_as="data.table")[]                                                                                                                               
                                                                                                                                                                                  
sdf <- tiledbsoma::SOMADataFrameOpen(uri,  tiledb_timestamp=ts3)                                                                                                                  
res <- sdf$read()$concat()                                                                                                                                                        
print(res)                                                                                                                                                                        
print(res$str)                                                                                                                                                                    
#tibble::as_tibble(res)                                                                                                                                                           
                           
```

</details>
> Worth testing on your patch.

@eddelbuettel before & after this PR I get the following when I run that:

```
Schema
soma_joinid: int64 not null
int: int32 not null
str: dictionary<values=string, indices=int8> not null
Error in self$open("WRITE", internal_use_only = "allowed_use") :
  tiledb_timestamp not yet supported for WRITE mode
Calls: <Anonymous> -> <Anonymous> -> <Anonymous> -> stopifnot
Execution halted
```

This PR should be merged to remove a known defect from the C++ implementation and we can continue from there.
@johnkerl when you said above

> [...] when I run that:

where you on `main` or on my branch adding this feature?  For me, on the branch, freshly rebased to the current main, all is now good.  I altered the test script to show a tibble as we now get a correct arrow dictionary and all is well:

```
[ ... ]
# A tibble: 5 × 3
  soma_joinid   int str  
        <int> <int> <fct>
1           6   106 c    
2           7   107 b    
3           8   108 c    
4           9   109 c    
5          10   110 b    
$ 
```

But that is on the de/sc-52516/timestamprange branch.
Good to know -- awesome @eddelbuettel ! :)
Branch should hopefully be ready "Real Soon Now (TM)" too.  
## [Codecov](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2895?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 89.98%. Comparing base [(`9f82dad`)](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/commit/9f82dad43da9ea3b8c09f066295900a7adb0e9f7?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) to head [(`d87cca3`)](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/commit/d87cca36bf547e1001e8e2b44442ad81ae284ffc?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data).
> Report is 4 commits behind head on main.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main    #2895      +/-   ##
==========================================
+ Coverage   89.83%   89.98%   +0.15%     
==========================================
  Files          37       37              
  Lines        3926     3926              
==========================================
+ Hits         3527     3533       +6     
+ Misses        399      393       -6     
```

| [Flag](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2895/flags?src=pr&el=flags&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) | Coverage Δ | |
|---|---|---|
| [python](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2895/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) | `89.98% <ø> (+0.15%)` | :arrow_up: |

Flags with carried forward coverage won't be shown. [Click here](https://docs.codecov.io/docs/carryforward-flags?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data#carryforward-flags-in-the-pull-request-comment) to find out more.

| [Components](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2895/components?src=pr&el=components&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) | Coverage Δ | |
|---|---|---|
| [python_api](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2895/components?src=pr&el=component&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) | `89.98% <ø> (+0.15%)` | :arrow_up: |
| [libtiledbsoma](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2895/components?src=pr&el=component&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) | `∅ <ø> (∅)` | |

</details>
Good.  That should also help for R where we documented this first,
There is something going on, since

```
["a","b"]  -- indices:
[0,1,0,0,0]]
```

should have been

```
["a","b"]  -- indices:
[0,1,0,1,1]]
```

since we wrote `'b', 'b'` at slots 3,4

```
    pydict = {}
    pydict["soma_joinid"] = [3, 4]
    pydict["foo"] = pd.Series(['b', 'b'], dtype='category')
    rb = pa.Table.from_pydict(pydict)
```
I've verified that ☝️ is a separate bug. Running this on a built checkout without this PR:

```
ubuntu@segge[prod][kerl/h2h][TileDB-SOMA]$ cat ts.py
import tiledb
import tiledbsoma as soma
import pyarrow as pa
import tempfile
import numpy as np
import pandas as pd
import time

with tempfile.TemporaryDirectory() as uri:

    asch = pa.schema([
        ("foo", pa.dictionary(pa.int8(), pa.large_string()))
    ])

    # Create at time 1
    soma.DataFrame.create(uri, schema=asch, tiledb_timestamp=1).close()

    # Write at time 2
    atbl = pa.Table.from_pydict({
        "soma_joinid": [0, 1, 2],
        "foo": pd.Series(['a', 'b', 'a'], dtype='category'),
    })
    with soma.DataFrame.open(uri, "w", tiledb_timestamp=2) as sdf:
        sdf.write(atbl)

    # Write at time 3
    atbl = pa.Table.from_pydict({
        "soma_joinid": [3, 4],
        #"foo": pd.Series(['b', 'c'], dtype='category'),
        "foo": pd.Series(['b', 'b'], dtype='category'),
    })
    with soma.DataFrame.open(uri, "w", tiledb_timestamp=3) as sdf:
        sdf.write(atbl)

    with soma.DataFrame.open(uri) as sdf:
        table = sdf.read().concat()
        print()
        print("TABLE AT NOW")
        print(table)
        print()
        print(table.to_pandas())
        print()
```

I get

```
TABLE AT NOW
pyarrow.Table
soma_joinid: int64
foo: dictionary<values=string, indices=int8, ordered=0>
----
soma_joinid: [[0,1,2,3,4]]
foo: [  -- dictionary:
["a","b"]  -- indices:
[0,1,0,0,0]]

   soma_joinid foo
0            0   a
1            1   b
2            2   a
3            3   a <------ note
4            4   a <------ note
```
> Good. That should also help for R where we documented this first,

@eddelbuettel great! If you have a ilnk for that "where" handy, please share it here -- that will be helpful.


Given #2896 and the proof above that the issue I identified is orthogonal to this PR (and pre-dates it), I think we should move forward with this in order to facilitate ongoing R-side development. And #2896 remains open.

I'll add a Python unit-test case as noted above.
> > Good. That should also help for R where we documented this first,
> > 
> @eddelbuettel great! If you have a ilnk for that "where" handy, please share it here -- that will be helpful.
> 

Never mind -- I found #2879
@johnkerl   Here is my actual test script from last week (which is not yet committed to the 'scraps' repo I use for these).  With the SOMA version I have here (a few days old) it just comes up empty (no 'strings' for the enumeration).  Worth testing on your patch.

<details>

```r
!#!/usr/bin/env Rscript                                                                                                                                                            
                                                                                                                                                                                  
uri <- "/tmp/tiledb/somadf-evolve-ts"                                                                                                                                             
if (dir.exists(uri)) unlink(uri, recursive=TRUE)                                                                                                                                  
                                                                                                                                                                                  
schema <- arrow::schema(arrow::field("soma_joinid", arrow::int64(), nullable = FALSE),                                                                                            
                        arrow::field("int", arrow::int32(), nullable = FALSE),                                                                                                    
                        arrow::field("str", arrow::dictionary(index_type = arrow::int8(), value_type = arrow::utf8(), ordered = FALSE), nullable=FALSE))                          
print(schema)                                                                                                                                                                     
ts1 <- rep(as.POSIXct(1, tz="UTC"), 2)                                                                                                                                            
sdf <- tiledbsoma::SOMADataFrameCreate(uri, schema, tiledb_timestamp=ts1[1])                                                                                                      
data <- arrow::arrow_table(soma_joinid = bit64::as.integer64(1L:5L),                                                                                                              
                           int = 101:105L,                                                                                                                                        
                           str = factor(c('a','b','b','a','b')))                                                                                                                  
cat("\n")                                                                                                                                                                         
print(tibble::as_tibble(data))                                                                                                                                                    
ts2 <- rep(as.POSIXct(2, tz="UTC"), 2)                                                                                                                                            
sdf$write(data, ts2)                                                                                                                                                              
                                                                                                                                                                                  
data <- arrow::arrow_table(soma_joinid = bit64::as.integer64(6L:10L),                                                                                                             
                           int = 106:110L,                                                                                                                                        
                           str = factor(c('c','b','c','c','b')))                                                                                                                  
cat("\n")                                                                                                                                                                         
print(tibble::as_tibble(data))                                                                                                                                                    
ts3 <- rep(as.POSIXct(3, tz="UTC"), 2)                                                                                                                                            
sdf$write(data, ts3)                                                                                                                                                              
sdf$close()                                                                                                                                                                       
                                                                                                                                                                                  
cat("\n")                                                                                                                                                                         
tiledb::tiledb_array(uri, return_as="data.table")[]                                                                                                                               
                                                                                                                                                                                  
sdf <- tiledbsoma::SOMADataFrameOpen(uri,  tiledb_timestamp=ts3)                                                                                                                  
res <- sdf$read()$concat()                                                                                                                                                        
print(res)                                                                                                                                                                        
print(res$str)                                                                                                                                                                    
#tibble::as_tibble(res)                                                                                                                                                           
                           
```

</details>
> Worth testing on your patch.

@eddelbuettel before & after this PR I get the following when I run that:

```
Schema
soma_joinid: int64 not null
int: int32 not null
str: dictionary<values=string, indices=int8> not null
Error in self$open("WRITE", internal_use_only = "allowed_use") :
  tiledb_timestamp not yet supported for WRITE mode
Calls: <Anonymous> -> <Anonymous> -> <Anonymous> -> stopifnot
Execution halted
```

This PR should be merged to remove a known defect from the C++ implementation and we can continue from there.
@johnkerl when you said above

> [...] when I run that:

where you on `main` or on my branch adding this feature?  For me, on the branch, freshly rebased to the current main, all is now good.  I altered the test script to show a tibble as we now get a correct arrow dictionary and all is well:

```
[ ... ]
# A tibble: 5 × 3
  soma_joinid   int str  
        <int> <int> <fct>
1           6   106 c    
2           7   107 b    
3           8   108 c    
4           9   109 c    
5          10   110 b    
$ 
```

But that is on the de/sc-52516/timestamprange branch.
Good to know -- awesome @eddelbuettel ! :)
Branch should hopefully be ready "Real Soon Now (TM)" too.  
## [Codecov](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2897?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 90.03%. Comparing base [(`7241fef`)](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/commit/7241fef757b87eba528c1ac02eb0c03f9ff5e1e7?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) to head [(`0ecb3fc`)](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/commit/0ecb3fc7aaae35fd7b3badf0f9002fae34f62076?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data).
> Report is 2 commits behind head on main.

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##             main    #2897      +/-   ##
==========================================
+ Coverage   89.88%   90.03%   +0.15%     
==========================================
  Files          37       37              
  Lines        3925     3925              
==========================================
+ Hits         3528     3534       +6     
+ Misses        397      391       -6     
```

| [Flag](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2897/flags?src=pr&el=flags&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) | Coverage Δ | |
|---|---|---|
| [python](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2897/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) | `90.03% <ø> (+0.15%)` | :arrow_up: |

Flags with carried forward coverage won't be shown. [Click here](https://docs.codecov.io/docs/carryforward-flags?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data#carryforward-flags-in-the-pull-request-comment) to find out more.

| [Components](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2897/components?src=pr&el=components&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) | Coverage Δ | |
|---|---|---|
| [python_api](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2897/components?src=pr&el=component&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) | `90.03% <ø> (+0.15%)` | :arrow_up: |
| [libtiledbsoma](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2897/components?src=pr&el=component&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) | `∅ <ø> (∅)` | |

</details>
## [Codecov](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2909?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) Report
Attention: Patch coverage is `50.00000%` with `4 lines` in your changes missing coverage. Please review.
> Project coverage is 59.52%. Comparing base [(`bdba3c3`)](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/commit/bdba3c38fb4d5ae01902ac4db93cdaa2d6a3d65c?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) to head [(`4a16241`)](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/commit/4a162411e2ac3befe2cbea51a968240561fea0e1?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data).
> Report is 4 commits behind head on main.

<details><summary>Additional details and impacted files</summary>


```diff
@@             Coverage Diff             @@
##             main    #2909       +/-   ##
===========================================
- Coverage   89.88%   59.52%   -30.37%     
===========================================
  Files          37       92       +55     
  Lines        3925    11337     +7412     
  Branches        0      724      +724     
===========================================
+ Hits         3528     6748     +3220     
- Misses        397     4418     +4021     
- Partials        0      171      +171     
```

| [Flag](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2909/flags?src=pr&el=flags&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) | Coverage Δ | |
|---|---|---|
| [libtiledbsoma](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2909/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) | `43.36% <50.00%> (?)` | |
| [python](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2909/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) | `90.03% <ø> (+0.15%)` | :arrow_up: |

Flags with carried forward coverage won't be shown. [Click here](https://docs.codecov.io/docs/carryforward-flags?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data#carryforward-flags-in-the-pull-request-comment) to find out more.

| [Components](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2909/components?src=pr&el=components&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) | Coverage Δ | |
|---|---|---|
| [python_api](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2909/components?src=pr&el=component&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) | `90.03% <ø> (+0.15%)` | :arrow_up: |
| [libtiledbsoma](https://app.codecov.io/gh/single-cell-data/TileDB-SOMA/pull/2909/components?src=pr&el=component&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=single-cell-data) | `50.00% <50.00%> (∅)` | |

</details>

Fix patch:
diff --git a/libtiledbsoma/src/soma/soma_array.cc b/libtiledbsoma/src/soma/soma_array.cc
index 32d415aa38..153bacd9b9 100644
--- a/libtiledbsoma/src/soma/soma_array.cc
+++ b/libtiledbsoma/src/soma/soma_array.cc
@@ -473,13 +473,6 @@ uint64_t SOMAArray::_get_max_capacity(tiledb_datatype_t index_type) {
 
 ArraySchemaEvolution SOMAArray::_make_se() {
     ArraySchemaEvolution se(*ctx_->tiledb_ctx());
-    if (timestamp_.has_value()) {
-        // ArraySchemaEvolution requires us to pair (t2, t2) even if our range
-        // is (t1, t2).
-        auto v = timestamp_.value();
-        TimestampRange tr(v.second, v.second);
-        se.set_timestamp_range(tr);
-    }
     return se;
 }
 


Test patch:
diff --git a/apis/python/tests/test_dataframe.py b/apis/python/tests/test_dataframe.py
index 514503f1fa..615a6ba4cd 100644
--- a/apis/python/tests/test_dataframe.py
+++ b/apis/python/tests/test_dataframe.py
@@ -1618,48 +1618,3 @@ def test_only_evolve_schema_when_enmr_is_extended(tmp_path):
     # subtract 1 for the __schema/__enumerations directory;
     # only looking at fragment files
     assert len(vfs.ls(os.path.join(uri, "__schema"))) - 1 == 3
-
-
-def test_timestamped_schema_evolve(tmp_path):
-    uri = tmp_path.as_posix()
-
-    asch = pa.schema([("myenum", pa.dictionary(pa.int8(), pa.large_string()))])
-
-    # Create at time 1
-    soma.DataFrame.create(uri, schema=asch, tiledb_timestamp=1).close()
-
-    # Write at time 2
-    atbl = pa.Table.from_pydict(
-        {
-            "soma_joinid": [0, 1, 2],
-            "myenum": pd.Series(["a", "b", "a"], dtype="category"),
-        }
-    )
-    with soma.DataFrame.open(uri, "w", tiledb_timestamp=2) as sdf:
-        sdf.write(atbl)
-
-    # Write at time 3
-    atbl = pa.Table.from_pydict(
-        {
-            "soma_joinid": [3, 4],
-            "myenum": pd.Series(["b", "b"], dtype="category"),
-        }
-    )
-    with soma.DataFrame.open(uri, "w", tiledb_timestamp=3) as sdf:
-        sdf.write(atbl)
-
-    with soma.DataFrame.open(uri, tiledb_timestamp=1) as sdf:
-        table = sdf.read().concat()
-        assert table["myenum"].to_pylist() == []
-
-    with soma.DataFrame.open(uri, tiledb_timestamp=2) as sdf:
-        table = sdf.read().concat()
-        assert table["myenum"].to_pylist() == ["a", "b", "a"]
-
-    with soma.DataFrame.open(uri, tiledb_timestamp=3) as sdf:
-        table = sdf.read().concat()
-        assert table["myenum"].to_pylist() == ["a", "b", "a", "b", "b"]
-
-    with soma.DataFrame.open(uri) as sdf:
-        table = sdf.read().concat()
-        assert table["myenum"].to_pylist() == ["a", "b", "a", "b", "b"]