Repository: JuliaML/MLUtils.jl
Pull request #6 (2021-12-28 12:14:59)

port batchview
Changes:
- `BatchView` loses the `count` keyword in the constructor
- it acquires the `partial` keyword, similarly to [Flux's Dataloader](https://github.com/FluxML/Flux.jl/blob/master/src/data/dataloader.jl)
- the default batch size is now 1


Patch:
diff --git a/src/MLUtils.jl b/src/MLUtils.jl
index 1527922..d0ad21f 100644
--- a/src/MLUtils.jl
+++ b/src/MLUtils.jl
@@ -19,7 +19,10 @@ include("splitobs.jl")
 export splitobs
 
 include("dataview.jl")
-export DataView, ObsView,
-       obsview
+export DataView,
+       obsview, ObsView,
+       batchview, BatchView, 
+       batchsize
+
 
 end
diff --git a/src/dataview.jl b/src/dataview.jl
index 1fef418..289f9cc 100644
--- a/src/dataview.jl
+++ b/src/dataview.jl
@@ -48,7 +48,7 @@ Aside from the `AbstractArray` interface following additional
 methods are provided:
 
 - **`getobs(data::ObsView, indices::AbstractVector)`** :
-    Returns a `Vector` of indivdual observations specified by
+    Returns a `Vector` of individual observations specified by
     `indices`.
 
 - **`numobs(data::ObsView)`** :
@@ -66,7 +66,7 @@ Examples
 =========
 
 ```julia
-X, Y = MLDataUtils.load_iris()
+X, Y = MLUtils.load_iris()
 
 A = obsview(X)
 @assert typeof(A) <: ObsView <: AbstractVector
@@ -121,6 +121,8 @@ Base.length(A::ObsView) = numobs(A)
 Base.getindex(A::ObsView, i::Int) = datasubset(A.data, i)
 Base.getindex(A::ObsView, i::AbstractVector) = ObsView(datasubset(A.data, i))
 
+datasubset(A::ObsView, i) = A[i]
+
 function Base.showarg(io::IO, A::ObsView, toplevel)
     print(io, "obsview(")
     Base.showarg(io, parent(A), false)
@@ -128,25 +130,159 @@ function Base.showarg(io::IO, A::ObsView, toplevel)
     toplevel && print(io, " with eltype ", nameof(eltype(A))) # simplify
 end
 
-# ------------------------------------------------------------
+# --------------------------------------------------------------------
+
+"""
+    BatchView(data, size; partial=true)
+    BatchView(data; size=1, partial=true)
+
+
+Description
+============
+
+Create a view of the given `data` that represents it as a vector
+of batches. Each batch will contain an equal amount of
+observations in them. The batch-size
+can be specified using the  parameter `size`.
+In the case that the size of the dataset is not dividable by the
+specified `size`, the remaining observations will
+be ignored if `partial=false`. If  `partial=true` instead 
+the last batch size can be slight smaller.
+
+Note that any data access is delayed until `getindex` is called,
+and even `getindex` returns the result of [`datasubset`](@ref)
+which in general avoids data movement until [`getobs`](@ref) is
+called.
+
+If used as an iterator, the object will iterate over the dataset
+once, effectively denoting an epoch. 
+
+Arguments
+==========
+
+- **`data`** : The object describing the dataset. Can be of any
+    type as long as it implements [`getobs`](@ref) and
+    [`numobs`](@ref) (see Details for more information).
+
+- **`size`** : The batch-size of each batch. I.e. the number of
+    observations that each batch must contain.
+
+- **`partial`** : If `partial=false` and the number of observations is 
+    not divisible by the batch size, then the last mini-batch is dropped.
+
+Details
+========
+
+For `BatchView` to work on some data structure, the type of the
+given variable `data` must implement the data container
+interface. See `?DataSubset` for more info.
+
+
+Examples
+=========
+
+```julia
+using MLUtils
+X, Y = MLUtils.load_iris()
+
+A = batchview(X, size = 30)
+@assert typeof(A) <: BatchView <: AbstractVector
+@assert eltype(A) <: SubArray{Float64,2}
+@assert length(A) == 5 # Iris has 150 observations
+@assert size(A[1]) == (4,30) # Iris has 4 features
+
+# 5 batches of size 30 observations
+for x in batchview(X, size = 30)
+    @assert typeof(x) <: SubArray{Float64,2}
+    @assert numobs(x) === 30
+end
+
+# 7 batches of size 20 observations
+# Note that the iris dataset has 150 observations,
+# which means that with a batchsize of 20, the last
+# 10 observations will be ignored
+for (x,y) in batchview((X,Y), size = 20, partial=false)
+    @assert typeof(x) <: SubArray{Float64,2}
+    @assert typeof(y) <: SubArray{String,1}
+    @assert numobs(x) === numobs(y) === 20
+end
+
+
+# randomly assign observations to one and only one batch.
+for (x,y) in batchview(shuffleobs((X,Y)), size=20)
+    @assert typeof(x) <: SubArray{Float64,2}
+    @assert typeof(y) <: SubArray{String,1}
+end
+```
+
+see also
+=========
+
+[`eachbatch`](@ref), [`ObsView`](@ref), [`shuffleobs`](@ref),
+[`getobs`](@ref), [`numobs`](@ref), [`DataSubset`](@ref)
+"""
+struct BatchView{TElem,TData} <: DataView{TElem,TData}
+    data::TData
+    size::Int
+    count::Int
+    partial::Bool
+    imax::Int
+end
+
+function BatchView(data::T; size::Int=1, partial::Bool=true) where {T}
+    # nsize, ncount = _compute_batch_settings(data, size, count, upto)
+    n = numobs(data)
+    if n < size
+        @warn "Number of observations less than batch size, decreasing the batch size to $n"
+        size = n
+    end
+    E = typeof(datasubset(data, 1:size))
+    imax = partial ? n : n - size + 1
+    count = partial ? ceil(Int, n / size) : floor(Int, n / size)
+    BatchView{E,T}(data, size, count, partial, imax)
+end
+
+"""
+    batchsize(data) -> Int
+
+Return the fixed size of each batch in `data`.
+"""
+batchsize(A::BatchView) = A.size
+numobs(A::BatchView) = A.count
+Base.parent(A::BatchView) = A.data
+Base.length(A::BatchView) = A.count
+Base.getindex(A::BatchView, batchindex::Int) =
+    datasubset(A.data, _batchrange(A, batchindex))
+
+function Base.getindex(A::BatchView, batchindices::AbstractVector)
+    obsindices = union((_batchrange(A, bi) for bi in batchindices)...)::Vector{Int}
+    BatchView(datasubset(A.data, obsindices); A.size, A.partial)
+end
+
+
+# Helper function to translate a batch-index into a range of observations.
+function _batchrange(a::BatchView, batchindex::Int)
+    (batchindex > a.count || batchindex < 0) && throw(BoundsError())
+    startidx = (batchindex - 1) * a.size + 1
+    endidx = min(numobs(a.data), startidx + a.size -1) 
+    return startidx:endidx
+end
+
+
+@doc (@doc BatchView)
+const batchview = BatchView
+
+datasubset(A::BatchView, i) = A[i]
 
+function Base.showarg(io::IO, A::BatchView, toplevel)
+    print(io, "batchview(")
+    Base.showarg(io, parent(A), false)
+    print(io, ", ")
+    print(io, "size=$(A.size), ")
+    print(io, "partial=$(A.partial)")
+    print(io, ')')
+    toplevel && print(io, " with eltype ", nameof(eltype(A))) # simplify
+end
 
 # --------------------------------------------------------------------
 
-# # if subsetting a DataView, then DataView the subset instead.
-# for fun in (:DataSubset, :datasubset)
-#     @eval @generated function ($fun)(A::T, i, obsdim::$O) where T<:AbstractObsView
-#         quote
-#             @assert obsdim == A.obsdim
-#             ($(T.name.name))(($($fun))(parent(A), i, obsdim), obsdim)
-#         end
-#     end
-#     @eval function ($fun)(A::BatchView, i, obsdim::$O)
-#         @assert obsdim == A.obsdim
-#         length(i) < A.size && throw(ArgumentError("The chosen batch-size ($(A.size)) is greater than the number of observations ($(length(i)))"))
-#         BatchView(($fun)(parent(A), i, obsdim), A.size, -1, obsdim)
-#     end
-# end
-
-DataSubset(A::ObsView, i) = ObsView(DataSubset(parent(A), i))
-datasubset(A::ObsView, i) = ObsView(datasubset(parent(A), i))


Test patch:
diff --git a/test/dataview.jl b/test/dataview.jl
index 18ab887..e42f01e 100644
--- a/test/dataview.jl
+++ b/test/dataview.jl
@@ -65,11 +65,10 @@
 
     @testset "subsetting" begin
         for var_raw in (vars..., tuples..., Xs, ys)
-        # for var_raw in (vars[1], )
             for var in (var_raw, DataSubset(var_raw))
-            # for var in (var_raw,)
                 A = ObsView(var)
                 @test getobs(@inferred(datasubset(A))) == @inferred(getobs(A))
+    
                 S = @inferred(datasubset(A, 1:5))
                 @test typeof(S) <: ObsView
                 @test @inferred(length(S)) == 5
@@ -77,12 +76,9 @@
                 @test @inferred(A[1:5]) == S
                 @test @inferred(getobs(A,1:5)) == getobs(S)
                 @test @inferred(getobs(S)) == getobs(ObsView(datasubset(var,1:5)))
+    
                 S = @inferred(DataSubset(A, 1:5))
-                @test typeof(S) <: ObsView
-                @test typeof(S.data) <: Union{DataSubset,Tuple}
-                @test @inferred(length(S)) == 5
-                @test @inferred(size(S)) == (5,)
-                @test @inferred(getobs(S)) == getobs(ObsView(DataSubset(var,1:5)))
+                @test typeof(S) <: DataSubset
             end
         end
         A = ObsView(X)
@@ -103,151 +99,109 @@
     end
 end
 
-# @testset "_compute_batch_settings" begin
-#     @test MLDataPattern._compute_batch_settings(X) === (30,5)
-#     @test MLDataPattern._compute_batch_settings(Xv) === (30,5)
-#     @test MLDataPattern._compute_batch_settings(Xs) === (30,5)
-#     @test MLDataPattern._compute_batch_settings(DataSubset(X)) === (30,5)
-#     @test MLDataPattern._compute_batch_settings((X,y)) === (30,5)
-#     @test MLDataPattern._compute_batch_settings((Xv,yv)) === (30,5)
-
-#     @test_throws ArgumentError MLDataPattern._compute_batch_settings(X, 160)
-#     @test_throws ArgumentError MLDataPattern._compute_batch_settings(X, 1, 160)
-#     @test_throws ArgumentError MLDataPattern._compute_batch_settings(X, 10, 20)
-
-#     for inner in (Xs, ys, vars...), var in (inner, DataSubset(inner))
-#         @test MLDataPattern._compute_batch_settings(var,10) === (10,15)
-#         @test MLDataPattern._compute_batch_settings(var,0,10) === (15,10)
-#         @test MLDataPattern._compute_batch_settings(var,-1,10) === (15,10)
-#         @test MLDataPattern._compute_batch_settings(var,10,10) === (10,15)
-#         @test MLDataPattern._compute_batch_settings(var,15,1) === (15,1)
-#         @test MLDataPattern._compute_batch_settings(var,15) === (15,1)
-#         @test MLDataPattern._compute_batch_settings(var,0,15) === (1,15)
-#         @test MLDataPattern._compute_batch_settings(var,-1,15) === (1,15)
-#     end
-# end
-
-# @testset "BatchView" begin
-#     @test BatchView <: AbstractVector
-#     @test BatchView <: LearnBase.DataView
-#     @test BatchView <: LearnBase.AbstractBatchView
-#     @test BatchView <: LearnBase.AbstractBatchIterator
-#     @test BatchView <: LearnBase.AbstractDataIterator
-#     @test batchview == BatchView
-#     @test_throws MethodError oversample(BatchView(X))
-#     @test_throws MethodError undersample(BatchView(X))
-#     @test_throws MethodError stratifiedobs(BatchView(X))
-
-#     @testset "constructor" begin
-#         @test_throws DimensionMismatch BatchView((rand(2,10),rand(9)))
-#         @test_throws DimensionMismatch BatchView((rand(2,10),rand(9)))
-#         @test_throws DimensionMismatch BatchView((rand(2,10),rand(4,9,10),rand(9)))
-#         @test_throws MethodError BatchView(EmptyType())
-#         for var in (vars..., tuples..., Xs, ys)
-#             @test_throws MethodError BatchView(var...)
-#             @test_throws ArgumentError BatchView(var, 151)
-#             A = BatchView(var, maxsize=151)
-#             @test length(A) == 1
-#             @test batchsize(A) == 15
-#             @test numobs(A) == numobs(var)
-#             @test @inferred(parent(BatchView(var))) === var
-#             A = @inferred(BatchView(var))
-#             @test @inferred(BatchView(A)) == A
-#             @test typeof(BatchView(A)) <: typeof(A)
-#             @test @inferred(BatchView(var)) == A
-#             @test numobs(A) == numobs(var)
-#             @test batchsize(A) == 30
-#         end
-#     end
+@testset "BatchView" begin
+    @test BatchView <: AbstractVector
+    @test BatchView <: DataView
+    @test batchview == BatchView
+    # @test_throws MethodError oversample(BatchView(X))
+    # @test_throws MethodError undersample(BatchView(X))
+    # @test_throws MethodError stratifiedobs(BatchView(X))
 
+    @testset "constructor" begin
+        @test_throws DimensionMismatch BatchView((rand(2,10),rand(9)))
+        @test_throws DimensionMismatch BatchView((rand(2,10),rand(9)))
+        @test_throws DimensionMismatch BatchView((rand(2,10),rand(4,9,10),rand(9)))
+        @test_throws MethodError BatchView(EmptyType())
+        for var in (vars..., tuples..., Xs, ys)
+            @test_throws MethodError BatchView(var...)
+            @test_throws MethodError BatchView(var, 16)
+            
+            A = BatchView(var, size=3)
+            @test length(A) == 5
+            @test batchsize(A) == 3
+            @test numobs(A) == length(A)
+            @test @inferred(parent(A)) === var
+        end
+        A = BatchView(X, size=16)
+        @test length(A) == 1
+        @test batchsize(A) == 15            
+    end
 
-#     @testset "typestability" begin
-#         for var in (vars..., tuples..., Xs, ys)
-#             @test typeof(@inferred(BatchView(var))) <: BatchView
-#             @test typeof(@inferred(BatchView(var, 10))) <: BatchView
-#             @test typeof(@inferred(BatchView(var, 10, 15))) <: BatchView
-#             @test typeof(@inferred(BatchView(var, -1, 10))) <: BatchView
-#             @test typeof(@inferred(BatchView(var))) <: BatchView
-#         end
-#         for tup in tuples
-#             @test typeof(@inferred(BatchView(tup))) <: BatchView
-#             @test typeof(@inferred(BatchView(tup, 5))) <: BatchView
-#             @test typeof(@inferred(BatchView(tup, -1, 10))) <: BatchView
-#         end
-#         @test typeof(@inferred(BatchView(CustomType()))) <: BatchView
-#     end
 
+    @testset "typestability" begin
+        for var in (vars..., tuples..., Xs, ys)
+            @test typeof(@inferred(BatchView(var))) <: BatchView
+            @test typeof(@inferred(BatchView(var, size=3))) <: BatchView
+            @test typeof(@inferred(BatchView(var, size=3, partial=true))) <: BatchView
+            @test typeof(@inferred(BatchView(var, size=3, partial=false))) <: BatchView
+        end
+        @test typeof(@inferred(BatchView(CustomType()))) <: BatchView
+    end
 
-#     @testset "AbstractArray interface" begin
-#         for var in (vars..., tuples..., Xs, ys)
-#             A = BatchView(var, 15)
-#             @test_throws BoundsError A[-1]
-#             @test_throws BoundsError A[11]
-#             @test @inferred(numobs(A)) == 15
-#             @test @inferred(length(A)) == 10
-#             @test @inferred(batchsize(A)) == 15
-#             @test @inferred(size(A)) == (10,)
-#             @test @inferred(getobs(A[2:3])) == getobs(BatchView(datasubset(var, 16:45), 15))
-#             @test @inferred(getobs(A[[1,3]])) == getobs(BatchView(datasubset(var, [1:15..., 31:45...]), 15))
-#             @test @inferred(A[1]) == datasubset(var, 1:15)
-#             @test @inferred(A[2]) == datasubset(var, 16:30)
-#             @test @inferred(A[3]) == datasubset(var, 31:45)
-#             @test A[end] == A[10]
-#             @test @inferred(getobs(A,1)) == getobs(var, 1:15)
-#             @test @inferred(getobs(A,2)) == getobs(var, 16:30)
-#             @test @inferred(getobs(A,3)) == getobs(var, 31:45)
-#             @test typeof(@inferred(collect(A))) <: Vector
-#         end
-#         for var in (vars..., tuples...)
-#             A = BatchView(var, 15)
-#             @test @inferred(getobs(A)) == A
-#             @test @inferred(A[2:3]) == BatchView(datasubset(var, 16:45), 15)
-#             @test @inferred(A[[1,3]]) == BatchView(datasubset(var, [1:15..., 31:45...]), 15)
-#         end
-#     end
+    @testset "AbstractArray interface" begin
+        for var in (vars..., tuples..., Xs, ys)
+            A = BatchView(var, size=5)
+            @test_throws BoundsError A[-1]
+            @test_throws BoundsError A[4]
+            @test @inferred(numobs(A)) == 3
+            @test @inferred(length(A)) == 3
+            @test @inferred(batchsize(A)) == 5
+            @test @inferred(size(A)) == (3,)
+            @test @inferred(getobs(A[2:3])) == getobs(BatchView(datasubset(var, 6:15), size=5))
+            @test @inferred(getobs(A[[1,3]])) == getobs(BatchView(datasubset(var, [1:5..., 11:15...]), size=5))
+            @test @inferred(A[1]) == datasubset(var, 1:5)
+            @test @inferred(A[2]) == datasubset(var, 6:10)
+            @test @inferred(A[3]) == datasubset(var, 11:15)
+            @test A[end] == A[3]
+            @test @inferred(getobs(A,1)) == getobs(var, 1:5)
+            @test @inferred(getobs(A,2)) == getobs(var, 6:10)
+            @test @inferred(getobs(A,3)) == getobs(var, 11:15)
+            @test typeof(@inferred(collect(A))) <: Vector
+        end
+        for var in (vars..., tuples...)
+            A = BatchView(var, size=5)
+            @test @inferred(getobs(A)) == A
+            @test @inferred(A[2:3]) == BatchView(datasubset(var, 6:15), size=5)
+            @test @inferred(A[[1,3]]) == BatchView(datasubset(var, [1:5..., 11:15...]), size=5)
+        end
+    end
 
 
-#     @testset "subsetting" begin
-#         for var in (vars..., tuples..., Xs, ys)
-#             A = BatchView(var)
-#             @test getobs(@inferred(datasubset(A))) == @inferred(getobs(A))
-#             @test_throws ArgumentError datasubset(A,1:5)
-#             S = @inferred(datasubset(A, 1:60))
-#             @test typeof(S) <: BatchView
-#             @test @inferred(numobs(S)) == 60
-#             @test @inferred(length(S)) == 2
-#             @test @inferred(size(S)) == (2,)
-#             @test getobs(@inferred(A[1:2])) == getobs(S)
-#             @test @inferred(getobs(A,1:2)) == getobs(S)
-#             @test @inferred(getobs(S)) == getobs(BatchView(datasubset(var,1:60),30))
-#             S = @inferred(DataSubset(A, 1:60))
-#             @test typeof(S) <: BatchView
-#             @test typeof(S.data) <: Union{DataSubset,Tuple}
-#             @test @inferred(length(S)) == 2
-#             @test @inferred(size(S)) == (2,)
-#             @test @inferred(getobs(S)) == getobs(BatchView(DataSubset(var,1:60),30))
-#         end
-#         A = BatchView(X)
-#         @test typeof(A.data) <: Array
-#         S = @inferred(datasubset(A))
-#         @test typeof(S) <: BatchView
-#         @test @inferred(length(S)) == 5
-#         @test typeof(S.data) <: SubArray
-#     end
+    @testset "subsetting" begin
+        for var in (vars..., tuples..., Xs, ys)
+            A = BatchView(var, size=3)
+            @test getobs(@inferred(datasubset(A))) == @inferred(getobs(A))
+            @test_throws BoundsError datasubset(A,1:6)
+            
+            S = @inferred(datasubset(A, 1:2))
+            @test typeof(S) <: BatchView
+            @test @inferred(numobs(S)) == 2
+            @test @inferred(length(S)) == numobs(S)
+            @test @inferred(size(S)) == (length(S),)
+            @test getobs(@inferred(A[1:2])) == getobs(S)
+            @test @inferred(getobs(A,1:2)) == getobs(S)
+            @test @inferred(getobs(S)) == getobs(BatchView(datasubset(var,1:6),size=3))
+            
+            S = @inferred(DataSubset(A, 1:2))
+            @test typeof(S) <: DataSubset
+        end
+        A = BatchView(X, size=3)
+        @test typeof(A.data) <: Array
+        S = @inferred(datasubset(A))
+        @test typeof(S) <: BatchView
+        @test @inferred(length(S)) == 5
+        @test typeof(S.data) <: SubArray
+    end
 
-#     @testset "nesting with ObsView" begin
-#         for var in vars
-#             @test eltype(@inferred(BatchView(ObsView(var)))[1]) <: Union{SubArray,String}
-#             @test @inferred(BatchView(BatchView(var))) == BatchView(var)
-#         end
-#         for var in tuples
-#             @test eltype(@inferred(BatchView(ObsView(var)))[1]) <: Tuple
-#             @test @inferred(BatchView(BatchView(var))) == BatchView(var)
-#         end
-#         for var in (Xs, ys)
-#             @test eltype(@inferred(BatchView(ObsView(var)))[1]) <: DataSubset
-#             @test @inferred(BatchView(BatchView(var))) == BatchView(var)
-#         end
-#         @test ObsView(BatchView(X)) == ObsView(X)
-#     end
-# end
+    @testset "nesting with ObsView" begin
+        for var in vars
+            @test eltype(@inferred(BatchView(ObsView(var)))[1]) <: Union{SubArray,String}
+        end
+        for var in tuples
+            @test eltype(@inferred(BatchView(ObsView(var)))[1]) <: Tuple
+        end
+        for var in (Xs, ys)
+            @test eltype(@inferred(BatchView(ObsView(var)))[1]) <: SubArray
+        end
+    end
+end