Repository: JuliaAI/MLJBase.jl
Pull request #632 (2021-09-06 04:51:31)

For a 0.18.20 release
#628 #627 


Patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 85c43219..173e5b30 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -19,7 +19,7 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - '1.0'
+          - '1.3' 
           - '1' # automatically expands to the latest stable 1.x release of Julia.
         os:
           - ubuntu-latest
diff --git a/Project.toml b/Project.toml
index f0a42057..75998d94 100644
--- a/Project.toml
+++ b/Project.toml
@@ -1,7 +1,7 @@
 name = "MLJBase"
 uuid = "a7f614a8-145f-11e9-1d2a-a57a1082229d"
 authors = ["Anthony D. Blaom <anthony.blaom@gmail.com>"]
-version = "0.18.19"
+version = "0.18.20"
 
 [deps]
 CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"
@@ -33,7 +33,7 @@ ComputationalResources = "0.3"
 Distributions = "0.22, 0.23, 0.24, 0.25"
 InvertedIndices = "1"
 LossFunctions = "0.5, 0.6, 0.7"
-MLJModelInterface = "1.2"
+MLJModelInterface = "1.3"
 Missings = "0.4, 1"
 OrderedCollections = "1.1"
 Parameters = "0.12"
@@ -43,7 +43,7 @@ ScientificTypes = "2.1"
 StatisticalTraits = "2.1"
 StatsBase = "0.32, 0.33"
 Tables = "0.2, 1.0"
-julia = "1"
+julia = "1.3"
 
 [extras]
 DecisionTree = "7806a523-6efd-50cb-b5f6-3fa6f1930dbb"
diff --git a/src/MLJBase.jl b/src/MLJBase.jl
index c2c83585..48e5141e 100644
--- a/src/MLJBase.jl
+++ b/src/MLJBase.jl
@@ -21,8 +21,8 @@ import StatisticalTraits.snakecase
 # Interface
 using MLJModelInterface
 import MLJModelInterface: fit, update, update_data, transform,
-    inverse_transform, fitted_params, predict,
-    predict_mode, predict_mean, predict_median, predict_joint,
+    inverse_transform, fitted_params, predict, predict_mode, 
+    predict_mean, predict_median, predict_joint,
     evaluate, clean!, is_same_except,
     save, restore, is_same_except, istransparent,
     params, training_losses
@@ -88,8 +88,8 @@ export @mlj_model, metadata_pkg, metadata_model
 
 # model api
 export fit, update, update_data, transform, inverse_transform,
-    fitted_params, predict,
-    predict_mode, predict_mean, predict_median, predict_joint,
+    fitted_params, predict, predict_mode, predict_mean, 
+    predict_median, predict_joint,
     evaluate, clean!, training_losses
 
 # data operations
@@ -173,7 +173,14 @@ export ResamplingStrategy, Holdout, CV, StratifiedCV, TimeSeriesCV,
 export measures, metadata_measure
 
 # measure/measures.jl (excluding traits):
-export aggregate, default_measure, value
+export aggregate, default_measure, value, skipinvalid
+
+# measures/probabilistic:
+export cross_entropy, BrierScore, brier_score,
+    BrierLoss, brier_loss,
+    LogLoss, log_loss, LogScore, log_score,
+    SphericalScore, spherical_score,
+    auc, area_under_curve, roc_curve, roc
 
 # measures/continuous.jl:
 export mav, mae, mape, rms, rmsl, rmslp1, rmsp, l1, l2, log_cosh,
@@ -188,18 +195,12 @@ export mav, mae, mape, rms, rmsl, rmslp1, rmsp, l1, l2, log_cosh,
 export confusion_matrix, confmat, ConfusionMatrix
 
 # measures/finite.jl:
-export cross_entropy, BrierScore, brier_score,
-    BrierLoss, brier_loss,
-    LogLoss, log_loss,
-    misclassification_rate, mcr, accuracy,
+export misclassification_rate, mcr, accuracy,
     balanced_accuracy, bacc, bac, BalancedAccuracy,
     matthews_correlation, mcc, MCC, AUC, AreaUnderCurve,
     MisclassificationRate, Accuracy, MCR, BACC, BAC,
     MatthewsCorrelation
 
-# measures/finite.jl -- Multiclass{2} (order independent):
-export auc, area_under_curve, roc_curve, roc
-
 # measures/finite.jl -- OrderedFactor{2} (order dependent):
 export TruePositive, TrueNegative, FalsePositive, FalseNegative,
     TruePositiveRate, TrueNegativeRate, FalsePositiveRate,
diff --git a/src/composition/learning_networks/machines.jl b/src/composition/learning_networks/machines.jl
index 330a35a9..f21d25b5 100644
--- a/src/composition/learning_networks/machines.jl
+++ b/src/composition/learning_networks/machines.jl
@@ -52,6 +52,38 @@ end
 
 caches_data_by_default(::Type{<:Surrogate}) = false
 
+const ERR_MUST_PREDICT = ArgumentError(
+    "You must specify at least `predict=<some node>`. ")
+const ERR_MUST_TRANSFORM = ArgumentError(
+    "You must specify at least `transform=<some node>`. ")
+const ERR_MUST_OPERATE = ArgumentError(
+    "You must specify at least one operation, as in `predict=<some node>`. ")
+const ERR_MUST_SPECIFY_SOURCES = ArgumentError(
+    "You must specify at least one source `Xs`, as in "*
+    "`machine(surrogate_model, Xs, ...; kwargs...)`. ")
+
+function check_surrogate_machine(::Surrogate, signature, _sources)
+    isempty(signature) && throw(ERR_MUST_OPERATE)
+    isempty(_sources) && throw(ERR_MUST_SPECIFY_SOURCES)
+    return nothing
+end
+
+function check_surrogate_machine(::Union{Supervised,SupervisedAnnotator},
+                                 signature,
+                                 _sources)
+    isempty(signature) && throw(ERR_MUST_PREDICT)
+    length(_sources) > 1 || throw(err_supervised_nargs())
+    return nothing
+end
+
+function check_surrogate_machine(::Union{Unsupervised},
+                                 signature,
+                                 _sources)
+    isempty(signature) && throw(ERR_MUST_TRANSFORM)
+    length(_sources) < 2 || throw(err_unsupervised_nargs())
+    return nothing
+end
+
 function machine(model::Surrogate, _sources::Source...; pair_itr...)
 
     # named tuple, such as `(predict=yhat, transform=W)`:
@@ -60,22 +92,8 @@ function machine(model::Surrogate, _sources::Source...; pair_itr...)
         op in OPERATIONS || throw(ArgumentError(
             "`$op` is not an admissible operation. "))
     end
-    if isempty(signature)
-        model isa Supervised &&
-            throw(ArgumentError(
-                "You must specify at least `predict=<some node>`. "))
-        model isa Unsupervised &&
-            throw(ArgumentError(
-                "You must specify at least `transform=<some node>`. "))
-    end
 
-    if model isa Supervised
-        length(_sources) > 1 || throw(err_supervised_nargs())
-    elseif model isa Unsupervised
-        length(_sources) < 2 || throw(err_unsupervised_nargs())
-    else
-        throw(DomainError)
-    end
+    check_surrogate_machine(model, signature, _sources)
 
     mach = Machine(model, _sources...)
 
@@ -85,26 +103,24 @@ function machine(model::Surrogate, _sources::Source...; pair_itr...)
 
 end
 
-function machine(sources::Source...; pair_itr...)
+function machine(_sources::Source...; pair_itr...)
 
     signature = (; pair_itr...)
 
-    isempty(signature) &&
-        throw(ArgumentError(
-            "You must specify at least `predict=<some node>` or "*
-            "or `transform=<some node>`. "))
+    isempty(signature) && throw(ERR_MUST_OPERATE)
 
     T = model_supertype(signature)
     if T == nothing
         @warn "Unable to infer surrogate model type. \n"*
-        "Using Deterministic(). To override use:\n "*
+            "Using Deterministic(). To override specify "*
+            "surrogate model, as in "*
         "`machine(Probabilistic(), ...)` or `machine(Interval(), ...)`"
         model = Deterministic()
     else
         model = surrogate(T)
     end
 
-    return machine(model, sources...; pair_itr...)
+    return machine(model, _sources...; pair_itr...)
 
 end
 
diff --git a/src/composition/learning_networks/nodes.jl b/src/composition/learning_networks/nodes.jl
index 7f56bbbc..fd7f8af6 100644
--- a/src/composition/learning_networks/nodes.jl
+++ b/src/composition/learning_networks/nodes.jl
@@ -457,6 +457,8 @@ Statistics.median(X::AbstractNode) = node(v->median.(v), X)
 StatsBase.mode(X::AbstractNode)    = node(v->mode.(v), X)
 Base.log(X::AbstractNode) = node(v->log.(v), X)
 Base.exp(X::AbstractNode) = node(v->exp.(v), X)
+Base.first(X::AbstractNode) = node(first, X)
+Base.last(X::AbstractNode) = node(last, X)
 
 +(y1::AbstractNode, y2::AbstractNode) = node(+, y1, y2)
 +(y1, y2::AbstractNode) = node(+, y1, y2)
diff --git a/src/data/data.jl b/src/data/data.jl
index f766dfc3..049e8eab 100644
--- a/src/data/data.jl
+++ b/src/data/data.jl
@@ -82,14 +82,19 @@ end
 
 """
     partition(X, fractions...;
-              shuffle=nothing, rng=Random.GLOBAL_RNG, stratify=nothing)
+              shuffle=nothing,
+              rng=Random.GLOBAL_RNG,
+              stratify=nothing)
 
-Splits the vector or matrix `X` into a tuple of vectors or matrices whose
-numbers of rows are given by the corresponding `fractions` of
-`length(nrows(X))`, where valid fractions are in (0,1) and sum up to less than
-1. The last fraction is not provided, as it is inferred from the preceding ones.
-`X` can also be any object which implements the `Tables.jl` interface according
-to `Tables.istable`.
+Splits the vector or matrix `X` into a tuple of vectors or matrices
+whose vertical concatentation is `X`. The number of rows in each
+componenent of the return value is determined by the corresponding
+`fractions` of `length(nrows(X))`, where valid fractions are in (0,1)
+and sum up to less than one. The last fraction is not provided, as it
+is inferred from the preceding ones.
+
+`X` can also be any object which implements the `Tables.jl` interface
+according to `Tables.istable`.
 
 So, for example,
 
@@ -102,6 +107,9 @@ So, for example,
     julia> partition(reshape(1:10, 5, 2), 0.2, 0.4)
     ([1 6], [2 7; 3 8], [4 9; 5 10])
 
+    X, _ = make_regression()          # a table
+    Xtrain, Xtest = partition(X, 0.8) # the table split on rows
+
 ## Keywords
 
 * `shuffle=nothing`:        if set to  `true`, shuffles the rows before taking fractions.
@@ -144,17 +152,17 @@ end
 """
 
     t1, t2, ...., tk = unpack(table, f1, f2, ... fk;
-                               wrap_singles=false,
-                               shuffle=false,
-                               rng::Union{AbstractRNG,Int,Nothing}=nothing)
-
-
-
-Split any Tables.jl compatible `table` into smaller tables (or
-vectors) `t1, t2, ..., tk` by making selections *without replacement*
-from the column names defined by the filters `f1`, `f2`, ...,
-`fk`. A *filter* is any object `f` such that `f(name)` is `true`
-or `false` for each column `name::Symbol` of `table`.
+                             wrap_singles=false,
+                             shuffle=false,
+                             rng::Union{AbstractRNG,Int,Nothing}=nothing)
+
+Horizontally split any Tables.jl compatible `table` into smaller
+tables (or vectors) `t1, t2, ..., tk` by making column selections
+**without replacement** by successively applying the columnn name
+filters `f1`, `f2`, ..., `fk`. A *filter* is any object `f` such that
+`f(name)` is `true` or `false` for each column `name::Symbol` of
+`table`. For example, use the filter `_ -> true` to pick up all
+remaining columns of the table.
 
 Whenever a returned table contains a single column, it is converted to
 a vector unless `wrap_singles=true`.
@@ -162,7 +170,7 @@ a vector unless `wrap_singles=true`.
 Scientific type conversions can be optionally specified (note
 semicolon):
 
-    unpack(table, t...; wrap_singles=false, col1=>scitype1, col2=>scitype2, ... )
+    unpack(table, t...; col1=>scitype1, col2=>scitype2, ... )
 
 If `shuffle=true` then the rows of `table` are first shuffled, using
 the global RNG, unless `rng` is specified; if `rng` is an integer, it
@@ -352,3 +360,55 @@ MLJModelInterface.transform(e::Union{CategoricalArray, CategoricalValue},
                             arg) = transform_(CategoricalArrays.pool(e), arg)
 MLJModelInterface.transform(e::CategoricalPool, arg) =
     transform_(e, arg)
+
+
+## SKIPPING MISSING AND NAN: skipinvalid
+
+const ERR_NOTHING_LEFT_TO_AGGREGATE = ErrorException(
+    "Trying to aggregrate an empty collection of measurements. Perhaps all "*
+    "measuresments are `missing` or `NaN`. ")
+
+_isnan(x) = false
+_isnan(x::Number) = isnan(x)
+
+skipnan(x) = Iterators.filter(!_isnan, x)
+
+"""
+    skipinvalid(itr)
+
+Return an iterator over the elements in `itr` skipping `missing` and
+`NaN` values. Behaviour is similar to [`skipmissing`](@ref).
+
+    skipinvalid(A, B)
+
+For vectors `A` and `B` of the same length, return a tuple of vectors
+`(A[mask], B[mask])` where `mask[i]` is `true` if and only if `A[i]`
+and `B[i]` are both valid (non-`missing` and non-`NaN`). Can also
+called on other iterators of matching length, such as arrays, but
+always returns a vector. Does not remove `Missing` from the element
+types if present in the original iterators.
+"""
+@inline function skipinvalid(v)
+    ret = v |> skipmissing |> skipnan
+    isempty(ret) && throw(ERR_NOTHING_LEFT_TO_AGGREGATE)
+    return ret
+end
+
+isinvalid(x) = ismissing(x) || _isnan(x)
+
+function skipinvalid(yhat, y)
+    mask = .!(isinvalid.(yhat) .| isinvalid.(y))
+    return yhat[mask], y[mask]
+end
+
+# TODO: refactor balanced accuracy to get rid of these:
+
+function _skipinvalid(yhat, y, w::Arr)
+    mask = .!(isinvalid.(yhat) .| isinvalid.(y))
+    return yhat[mask], y[mask], w[mask]
+end
+
+function _skipinvalid(yhat, y, w::Union{Nothing,AbstractDict})
+    mask = .!(isinvalid.(yhat) .| isinvalid.(y))
+    return yhat[mask], y[mask], w
+end
diff --git a/src/machines.jl b/src/machines.jl
index 84b54376..7bd8e91e 100644
--- a/src/machines.jl
+++ b/src/machines.jl
@@ -114,7 +114,7 @@ err_length_mismatch(model::Supervised) = DimensionMismatch(
     "Differing number of observations "*
     "in input and target. ")
 
-check(model::Any, args...; kwargs) =
+check(model::Any, args...; kwargs...) =
     throw(ArgumentError("Expected a `Model` instance, got $model. "))
 
 function check(model::Model, args...; full=false)
diff --git a/src/measures/README.md b/src/measures/README.md
new file mode 100644
index 00000000..991fb303
--- /dev/null
+++ b/src/measures/README.md
@@ -0,0 +1,131 @@
+## Adding new measures
+
+A measure is ordinarily called on data directly, as in
+
+```julia
+ŷ = rand(3) # predictions
+y = rand(3) # ground truth observations
+
+m = LPLoss(p=3)
+
+julia> m(ŷ, y)
+3-element Vector{Float64}:
+ 0.07060087052171798
+ 0.003020044780949528
+ 0.019067038457889922
+```
+
+Recall that to call a measure without performing dimension or pool checks, one
+uses `MLJBase.call` instead:
+
+```julia
+MLJBase.call(m, ŷ, y)
+```
+
+A new measure reporting an aggregate measurement, such as
+`AreaUnderCurve`, will subtype `Aggregate`, and only needs to
+implement `call`. A measure that reports a measurement for each
+observation , such as `LPLoss`, subtypes `Unaggregated` and only needs
+to implement an evaluation method for single observations called
+`single`.
+
+Recall also that if a measure reports each observation, it does so
+even in the case that weights are additionally specified:
+
+```julia
+w = rand(3) # per-observation weights
+
+julia> m(ŷ, y, rand(3))
+3-element Vector{Float64}:
+ 0.049333392516241206
+ 0.0017612002314472718
+ 0.003157450446692638
+ ```
+
+This behaviour differs from other places where weights can only be
+specified as part of an aggregation of multi-observation measurements.
+
+
+### Unaggregated measures implement `single`
+
+To implement an `Unaggregated` measure, it suffices to implement
+`single(measure, η̂, η)`, which should return a measurement (e.g., a
+float) for a single example `(η̂, η)` (e.g., a pair of
+floats). Behavior on `missing` values is handled by fallbacks:
+
+```julia
+single(::Unaggregated, η̂::Missing, η)          = missing
+single(::Unaggregated, η̂,          η::Missing) = missing
+```
+
+Be sure to annotate the type of `η̂` and `η` in the implementation
+of `single` to avoid a type ambiguity with these fallbacks. For example
+
+```julia
+single(::MyUnaggregatedMeasure, η̂::Real, η::Real) = ...
+```
+or
+
+```
+single(::MyAggregatedMeasure, η̂::UnivariateFinite, η::Label) = ...
+```
+
+Here `Label` is just a large union type defined in MLJBase excluding
+`missing` as an instance:
+
+```julia
+const Label = Union{CategoricalValue,Number,AbstractString,Symbol,AbstractChar}
+```
+
+If only `single` is implemented, then the measure will automatically
+support per-observation weights and, where that makes sense, per-class
+weights. However, `supports_class_weights` may need to be overloaded,
+as this defaults to `false`.
+
+#### Special cases
+
+If `single` is *not* implemented, then `call(measure, ŷ, y)`, and
+optionally `call(measure, ŷ, y, w)`, must be implemented (the
+fallbacks call `single`).  In this case `y` and `ŷ` are arrays of
+matching size and the method should return an array of that size
+*without performing size or pool checks*. The method should handle
+`missing` and `NaN` values if possible, which should be propagated to
+relevant elements of the returned array.
+
+The `supports_weights` trait, which defaults to `true`, will need to
+be overloaded to return `false` if neither `single(::MyMeasure,
+args...)` nor `call(::MyMeasure, ŷ, y, w::AbstractArray)` are
+overloaded.
+
+### Aggregated measures implement `call`
+
+To implement an `Aggregated` measure, implement
+`call(measure::MyMeasure, ŷ, y)`. Optionally implement 
+`call(measure::MyMeasure, ŷ, y, w)`.
+
+
+### Trait declarations 
+
+Measure traits can be set using the `metadata_measure`
+function (query the doc-string) or individually, as in 
+
+```julia
+supports_weights(::Type{<:MyMeasure}) = false
+```
+
+Defaults are shown below
+
+trait                    | allowed values               | default 
+-------------------------|------------------------------|--------------
+`target_scitype`         | some scientific type         | `Unknown`
+`human_name`             | any `String`                 | string version of type name
+`instances`              | any `Vector{String}`         | empty
+`prediction_type`        | `:deterministic`, `:probabilistic`, `:interval` `:unknown` | `:unknown`
+`orientation`            | `:score`, `:loss`, `:unknown`| `:unknown`
+`aggregation`            | `Mean()`, `Sum()`, `RootMeanSqaure()` | `Mean()`
+`supports_weights`       | `true` or `false`            | `true`
+`supports_class_weights` | `true` or `false`            | `false`
+`docstring`              | any `String`                 | includes `name`, `human_name` and `instances`
+`distribution_type`      | any `Distribution` subtype or `Unknown`   | `Unknown`
+
+
diff --git a/src/measures/confusion_matrix.jl b/src/measures/confusion_matrix.jl
index 99e6269a..42d5adf4 100644
--- a/src/measures/confusion_matrix.jl
+++ b/src/measures/confusion_matrix.jl
@@ -31,10 +31,23 @@ end
 # allow to access cm[i,j] but not set (it's immutable)
 Base.getindex(cm::ConfusionMatrixObject, inds...) = getindex(cm.mat, inds...)
 
+_levels(y1, y2) = vcat(levels(y1), levels(y2)) |> unique
+
+# simultaneous coercion of two vectors into categorical vectors having
+# the same pool:
+function _categorical(y1, y2)
+    L = _levels(y1, y2)
+    return categorical(y1, levels=L), categorical(y2, levels=L)
+end
+_categorical(y1::CategoricalArray{V1,N},
+             y2::CategoricalArray{V2,N}) where
+    {V, V1<:Union{Missing,V}, V2<:Union{Missing,V}, N} =
+    y1, y2
+
 """
     _confmat(ŷ, y; rev=false)
 
-A private method. General users should use `confmat` or other instance
+A private method. General users should use `confmat` or other instances
 of the measure type [`ConfusionMatrix`](@ref).
 
 Computes the confusion matrix given a predicted `ŷ` with categorical elements
@@ -66,13 +79,15 @@ there no way to specify an ordering different from `levels(y)`, where
 `y` is the target.
 
 """
-function _confmat(ŷm::CatArrMissing{T,N}, ym::CatArrMissing{T,N};
+function _confmat(ŷraw::Union{Arr{V1,N}, CategoricalArray{V1,N}},
+                  yraw::Union{Arr{V2,N}, CategoricalArray{V2,N}};
                   rev::Union{Nothing,Bool}=nothing,
                   perm::Union{Nothing,Vector{<:Integer}}=nothing,
-                  warn::Bool=true) where {T,N}
+                  warn::Bool=true) where
+    {V,V1<:Union{Missing,V}, V2<:Union{Missing,V},N}
 
-    check_dimensions(ŷm, ym)
-    ŷ, y = _skipmissing(ŷm, ym)
+    # no-op if vectors already categorical arrays:
+    ŷ, y = _categorical(ŷraw, yraw)
 
     levels_ = levels(y)
     nc = length(levels_)
@@ -91,15 +106,19 @@ function _confmat(ŷm::CatArrMissing{T,N}, ym::CatArrMissing{T,N};
 
     # warning
     if rev === nothing && perm === nothing
+        S = nonmissingtype(elscitype(y))
         if warn &&
-            if nc==2 && !(scitype_union(y) >: OrderedFactor{2})
+            if nc==2 &&  !(S <: OrderedFactor)
                 @warn "The classes are un-ordered,\n" *
-                      "using: negative='$(levels_[1])' and positive='$(levels_[2])'.\n" *
-                      "To suppress this warning, consider coercing to OrderedFactor."
-            elseif !(scitype_union(y) >: OrderedFactor{nc})
+                    "using: negative='$(levels_[1])' "*
+                    "and positive='$(levels_[2])'.\n" *
+                    "To suppress this warning, consider coercing "*
+                    "to OrderedFactor."
+            elseif !(S <: OrderedFactor)
                 @warn "The classes are un-ordered,\n" *
                       "using order: $([l for l in levels_]).\n" *
-                      "To suppress this warning, consider coercing to OrderedFactor."
+                      "To suppress this warning, consider "*
+                      "coercing to OrderedFactor."
             end
         end
         rev  = false
@@ -117,6 +136,7 @@ function _confmat(ŷm::CatArrMissing{T,N}, ym::CatArrMissing{T,N};
     if isempty(perm)
         cmat = zeros(Int, nc, nc)
         @inbounds for i in eachindex(y)
+            (isinvalid(y[i]) || isinvalid(ŷ[i])) && continue
             cmat[int(ŷ[i]), int(y[i])] += 1
         end
         return ConfusionMatrixObject(cmat, string.(levels_))
@@ -126,6 +146,7 @@ function _confmat(ŷm::CatArrMissing{T,N}, ym::CatArrMissing{T,N};
     cmat = zeros(Int, nc, nc)
     iperm = invperm(perm)
     @inbounds for i in eachindex(y)
+        (isinvalid(y[i]) || isinvalid(ŷ[i])) && continue
         cmat[iperm[int(ŷ[i])], iperm[int(y[i])]] += 1
     end
     return ConfusionMatrixObject(cmat, string.(levels_[perm]))
@@ -196,7 +217,7 @@ end
 
 ## CONFUSION MATRIX AS MEASURE
 
-struct ConfusionMatrix <: Measure
+struct ConfusionMatrix <: Aggregated
     perm::Union{Nothing,Vector{<:Integer}}
 end
 
@@ -206,7 +227,8 @@ is_measure(::ConfusionMatrix) = true
 is_measure_type(::Type{ConfusionMatrix}) = true
 human_name(::Type{<:ConfusionMatrix}) = "confusion matrix"
 target_scitype(::Type{ConfusionMatrix}) =
-    AbstractVector{<:Union{Missing,Finite}}
+    Union{AbstractVector{<:Union{Missing,OrderedFactor}},
+          AbstractVector{<:Union{Missing,OrderedFactor}}}
 supports_weights(::Type{ConfusionMatrix}) = false
 prediction_type(::Type{ConfusionMatrix}) = :deterministic
 instances(::Type{<:ConfusionMatrix}) = ["confusion_matrix", "confmat"]
@@ -227,12 +249,12 @@ The ordering follows that of `levels(y)`.
 Use `ConfusionMatrix(perm=[2, 1])` to reverse the class order for binary
 data. For more than two classes, specify an appropriate permutation, as in
 `ConfusionMatrix(perm=[2, 3, 1])`.
+
 """,
 scitype=DOC_ORDERED_FACTOR_BINARY)
 
 # calling behaviour:
-(m::ConfusionMatrix)(ŷ, y) =
-    _confmat(ŷ, y, perm=m.perm)
+call(m::ConfusionMatrix, ŷ, y) = _confmat(ŷ, y, perm=m.perm)
 
 # overloading addition to make aggregation work:
 Base.round(m::MLJBase.ConfusionMatrixObject; kws...) = m
diff --git a/src/measures/continuous.jl b/src/measures/continuous.jl
index c2a87e29..d1ae819c 100644
--- a/src/measures/continuous.jl
+++ b/src/measures/continuous.jl
@@ -1,20 +1,18 @@
-# ===========================================================
-## DETERMINISTIC PREDICTIONS
+const InfiniteArrMissing = Union{
+    AbstractArray{<:Union{Missing,Continuous}},
+    AbstractArray{<:Union{Missing,Count}}}
 
 # -----------------------------------------------------------
 # MeanAbsoluteError
 
-struct MeanAbsoluteError <: Measure end
+struct MeanAbsoluteError <: Aggregated end
 
 metadata_measure(MeanAbsoluteError;
                  instances = ["mae", "mav", "mean_absolute_error",
                               "mean_absolute_value"],
-                 target_scitype           = Union{Vec{Continuous},Vec{Count}},
+                 target_scitype           = InfiniteArrMissing,
                  prediction_type          = :deterministic,
-                 orientation              = :loss,
-                 reports_each_observation = false,
-                 is_feature_dependent     = false,
-                 supports_weights         = true)
+                 orientation              = :loss),
 
 const MAE = MeanAbsoluteError
 const MAV = MeanAbsoluteError
@@ -27,43 +25,27 @@ body=
 ``\\text{mean absolute error} = n^{-1}∑ᵢwᵢ|yᵢ-ŷᵢ|``
 """)
 
-function (::MeanAbsoluteError)(ŷ::Vec{<:Real}, y::Vec{<:Real})
-    check_dimensions(ŷ, y)
-    ret = zero(eltype(y))
-    for i in eachindex(y)
-        dev = abs(y[i] - ŷ[i])
-        ret += dev
-    end
-    return ret / length(y)
-end
+call(::MeanAbsoluteError, ŷ::ArrMissing{<:Real}, y::ArrMissing{<:Real}) =
+    abs.(ŷ .- y) |> skipinvalid |> mean
 
-function (::MeanAbsoluteError)(ŷ::Vec{<:Real}, y::Vec{<:Real},
-                 w::Vec{<:Real})
-    check_dimensions(ŷ, y)
-    check_dimensions(y, w)
-    ret = zero(eltype(y))
-    for i in eachindex(y)
-        dev = abs(y[i] - ŷ[i])
-        ret += w[i]*dev
-    end
-    return ret / length(y)
-end
+call(::MeanAbsoluteError,
+     ŷ::ArrMissing{<:Real},
+     y::ArrMissing{<:Real},
+     w::Arr{<:Real}) =
+         abs.(ŷ .- y) .* w |> skipinvalid |> mean
 
 # ----------------------------------------------------------------
 # RootMeanSquaredError
 
-struct RootMeanSquaredError <: Measure end
+struct RootMeanSquaredError <: Aggregated end
 
 metadata_measure(RootMeanSquaredError;
                  instances                = ["rms", "rmse",
                                              "root_mean_squared_error"],
-                 target_scitype           = Union{Vec{Continuous},Vec{Count}},
+                 target_scitype           = InfiniteArrMissing,
                  prediction_type          = :deterministic,
                  orientation              = :loss,
-                 reports_each_observation = false,
-                 aggregation              = RootMeanSquare(),
-                 is_feature_dependent     = false,
-                 supports_weights         = true)
+                 aggregation              = RootMeanSquare())
 
 const RMS = RootMeanSquaredError
 @create_aliases RootMeanSquaredError
@@ -75,31 +57,19 @@ body=
 ``\\text{root mean squared error} = \\sqrt{\\frac{∑ᵢwᵢ|yᵢ-ŷᵢ|^2}{∑ᵢwᵢ}}``
 """)
 
-function (::RootMeanSquaredError)(ŷ::Vec{<:Real}, y::Vec{<:Real})
-    check_dimensions(ŷ, y)
-    ret = zero(eltype(y))
-    for i in eachindex(y)
-        dev = (y[i] - ŷ[i])^2
-        ret += dev
-    end
-    return sqrt(ret / length(y))
-end
+call(::RootMeanSquaredError, ŷ::ArrMissing{<:Real}, y::ArrMissing{<:Real}) =
+    (y .- ŷ).^2 |> skipinvalid |> mean |> sqrt
 
-function (::RootMeanSquaredError)(ŷ::Vec{<:Real}, y::Vec{<:Real},
-                 w::Vec{<:Real})
-    check_dimensions(ŷ, y)
-    ret = zero(eltype(y))
-    for i in eachindex(y)
-        dev = (y[i] - ŷ[i])^2
-        ret += w[i]*dev
-    end
-    return sqrt(ret / length(y))
-end
+call(::RootMeanSquaredError,
+     ŷ::ArrMissing{<:Real},
+     y::ArrMissing{<:Real},
+     w::Arr{<:Real}) =
+         (y .- ŷ).^2 .* w |> skipinvalid |> mean |> sqrt
 
 # -------------------------------------------------------------------
 # LP
 
-struct LPLoss{T<:Real} <: Measure
+struct LPLoss{T<:Real} <: Unaggregated
     p::T
 end
 
@@ -107,12 +77,9 @@ LPLoss(; p=2.0) = LPLoss(p)
 
 metadata_measure(LPLoss;
                  instances = ["l1", "l2"],
-                 target_scitype           = Union{Vec{Continuous},Vec{Count}},
+                 target_scitype           = InfiniteArrMissing,
                  prediction_type          = :deterministic,
-                 orientation              = :loss,
-                 reports_each_observation = true,
-                 is_feature_dependent     = false,
-                 supports_weights         = true)
+                 orientation              = :loss)
 
 const l1 = LPLoss(1)
 const l2 = LPLoss(2)
@@ -124,32 +91,19 @@ Constructor signature: `LPLoss(p=2)`. Reports
 `|ŷ[i] - y[i]|^p` for every index `i`.
 """)
 
-function (m::LPLoss)(ŷ::Vec{<:Real}, y::Vec{<:Real})
-    check_dimensions(ŷ, y)
-    return abs.((y - ŷ)).^(m.p)
-end
-
-function (m::LPLoss)(ŷ::Vec{<:Real}, y::Vec{<:Real},
-                w::Vec{<:Real})
-    check_dimensions(ŷ, y)
-    check_dimensions(w, y)
-    return w .* abs.((y - ŷ)).^(m.p)
-end
+single(m::LPLoss, ŷ::Real, y::Real) =  abs(y - ŷ)^(m.p)
 
 # ----------------------------------------------------------------------------
 # RootMeanSquaredLogError
 
-struct RootMeanSquaredLogError <: Measure end
+struct RootMeanSquaredLogError <: Aggregated end
 
 metadata_measure(RootMeanSquaredLogError;
                  instances = ["rmsl", "rmsle", "root_mean_squared_log_error"],
-                 target_scitype           = Union{Vec{Continuous},Vec{Count}},
+                 target_scitype           = InfiniteArrMissing,
                  prediction_type          = :deterministic,
                  orientation              = :loss,
-                 reports_each_observation = false,
-                 aggregation              = RootMeanSquare(),
-                 is_feature_dependent     = false,
-                 supports_weights         = false)
+                 aggregation              = RootMeanSquare())
 
 const RMSL = RootMeanSquaredLogError
 @create_aliases RootMeanSquaredLogError
@@ -162,20 +116,19 @@ n^{-1}∑ᵢ\\log\\left({yᵢ \\over ŷᵢ}\\right)``
 """,
 footer="See also [`rmslp1`](@ref).")
 
-function (::RootMeanSquaredLogError)(ŷ::Vec{<:Real}, y::Vec{<:Real})
-    check_dimensions(ŷ, y)
-    ret = zero(eltype(y))
-    for i in eachindex(y)
-        dev = (log(y[i]) - log(ŷ[i]))^2
-        ret += dev
-    end
-    return sqrt(ret / length(y))
-end
+call(::RootMeanSquaredLogError, ŷ::ArrMissing{<:Real}, y::ArrMissing{<:Real}) =
+    (log.(y) - log.(ŷ)).^2 |> skipinvalid |> mean |> sqrt
+
+call(::RootMeanSquaredLogError,
+      ŷ::ArrMissing{<:Real},
+      y::ArrMissing{<:Real},
+      w::Arr{<:Real}) =
+          (log.(y) - log.(ŷ)).^2 .* w |> skipinvalid |> mean |> sqrt
 
 # ---------------------------------------------------------------------------
 #  RootMeanSquaredLogProportionalError
 
-struct RootMeanSquaredLogProportionalError{T<:Real} <: Measure
+struct RootMeanSquaredLogProportionalError{T<:Real} <: Aggregated
     offset::T
 end
 
@@ -184,13 +137,10 @@ RootMeanSquaredLogProportionalError(; offset=1.0) =
 
 metadata_measure(RootMeanSquaredLogProportionalError;
                  instances                = ["rmslp1", ],
-                 target_scitype           = Union{Vec{Continuous},Vec{Count}},
+                 target_scitype           = InfiniteArrMissing,
                  prediction_type          = :deterministic,
                  orientation              = :loss,
-                 reports_each_observation = false,
-                 aggregation              = RootMeanSquare(),
-                 is_feature_dependent     = false,
-                 supports_weights         = false)
+                 aggregation              = RootMeanSquare())
 
 const RMSLP = RootMeanSquaredLogProportionalError
 @create_aliases RootMeanSquaredLogProportionalError
@@ -205,20 +155,18 @@ n^{-1}∑ᵢ\\log\\left({yᵢ + \\text{offset} \\over ŷᵢ + \\text{offset}}\\
 """,
 footer="See also [`rmsl`](@ref). ")
 
-function (m::RMSLP)(ŷ::Vec{<:Real}, y::Vec{<:Real})
-    check_dimensions(ŷ, y)
-    ret = zero(eltype(y))
-    for i in eachindex(y)
-        dev = (log(y[i] + m.offset) - log(ŷ[i] + m.offset))^2
-        ret += dev
-    end
-    return sqrt(ret / length(y))
-end
+call(m::RMSLP, ŷ::ArrMissing{<:Real}, y::ArrMissing{<:Real}) =
+    (log.(y .+ m.offset) - log.(ŷ .+ m.offset)).^2 |>
+    skipinvalid |> mean |> sqrt
+
+call(m::RMSLP, ŷ::ArrMissing{<:Real}, y::ArrMissing{<:Real}, w::Arr{<:Real}) =
+    (log.(y .+ m.offset) - log.(ŷ .+ m.offset)).^2 .* w |>
+    skipinvalid |> mean |> sqrt
 
 # --------------------------------------------------------------------------
 # RootMeanSquaredProportionalError
 
-struct RootMeanSquaredProportionalError{T<:Real} <: Measure
+struct RootMeanSquaredProportionalError{T<:Real} <: Aggregated
     tol::T
 end
 
@@ -227,15 +175,12 @@ RootMeanSquaredProportionalError(; tol=eps()) =
 
 metadata_measure(RootMeanSquaredProportionalError;
     instances                = ["rmsp", ],
-    target_scitype           = Union{Vec{Continuous},Vec{Count}},
+    target_scitype           = InfiniteArrMissing,
     prediction_type          = :deterministic,
     orientation              = :loss,
-    reports_each_observation = false,
-    aggregation              = RootMeanSquare(),
-    is_feature_dependent     = false,
-    supports_weights         = false)
+    aggregation              = RootMeanSquare())
 
-const RMSP = RootMeanSquaredProportionalError
+      const RMSP = RootMeanSquaredProportionalError
 @create_aliases RMSP
 
 @create_docs(RootMeanSquaredProportionalError,
@@ -251,15 +196,19 @@ of such indices.
 
 """)
 
-function (m::RootMeanSquaredProportionalError)(ŷ::Vec{<:Real}, y::Vec{<:Real})
-    check_dimensions(ŷ, y)
-    ret = zero(eltype(y))
+function call(m::RootMeanSquaredProportionalError,
+               ŷ::ArrMissing{<:Real},
+               y::ArrMissing{T},
+               w::Union{Nothing,Arr{<:Real}}=nothing) where T <: Real
+    ret = zero(T)
     count = 0
     @inbounds for i in eachindex(y)
+        (isinvalid(y[i]) || isinvalid(ŷ[i])) && continue
         ayi = abs(y[i])
         if ayi > m.tol
             dev = ((y[i] - ŷ[i]) / ayi)^2
             ret += dev
+            ret = _scale(ret, w, i)
             count += 1
         end
     end
@@ -269,7 +218,7 @@ end
 # -----------------------------------------------------------------------
 # MeanAbsoluteProportionalError
 
-struct MeanAbsoluteProportionalError{T} <: Measure
+struct MeanAbsoluteProportionalError{T} <: Aggregated
     tol::T
 end
 
@@ -277,14 +226,9 @@ MeanAbsoluteProportionalError(; tol=eps()) = MeanAbsoluteProportionalError(tol)
 
 metadata_measure(MeanAbsoluteProportionalError;
     instances                = ["mape", ],
-    target_scitype           = Union{Vec{Continuous},Vec{Count}},
+    target_scitype           = InfiniteArrMissing,
     prediction_type          = :deterministic,
-    orientation              = :loss,
-    reports_each_observation = false,
-    is_feature_dependent     = false,
-    supports_weights         = false,
-    docstring                = "Mean Absolute Proportional Error; "*
-                 "aliases: `mape=MAPE()`.")
+    orientation              = :loss)
 
 const MAPE = MeanAbsoluteProportionalError
 @create_aliases MAPE
@@ -300,17 +244,20 @@ where the sum is over indices such that `abs(yᵢ) > tol` and `m` is the number
 of such indices.
 """)
 
-function (m::MeanAbsoluteProportionalError)(ŷ::Vec{<:Real}, y::Vec{<:Real})
-    check_dimensions(ŷ, y)
-    ret = zero(eltype(y))
+function call(m::MeanAbsoluteProportionalError,
+              ŷ::ArrMissing{<:Real},
+              y::ArrMissing{T},
+              w::Union{Nothing,Arr{<:Real}}=nothing) where T <: Real
+    ret = zero(T)
     count = 0
     @inbounds for i in eachindex(y)
+        (isinvalid(y[i]) || isinvalid(ŷ[i])) && continue
         ayi = abs(y[i])
         if ayi > m.tol
         #if y[i] != zero(eltype(y))
             dev = abs((y[i] - ŷ[i]) / ayi)
-            #dev = abs((y[i] - ŷ[i]) / y[i])
             ret += dev
+            ret =_scale(ret, w, i)
             count += 1
         end
     end
@@ -320,17 +267,13 @@ end
 # -------------------------------------------------------------------------
 # LogCoshLoss
 
-struct LogCoshLoss <: Measure end
+struct LogCoshLoss <: Unaggregated end
 
 metadata_measure(LogCoshLoss;
     instances                = ["log_cosh", "log_cosh_loss"],
-    target_scitype           = Union{Vec{Continuous},Vec{Count}},
+    target_scitype           = InfiniteArrMissing,
     prediction_type          = :deterministic,
-    orientation              = :loss,
-    reports_each_observation = true,
-    is_feature_dependent     = false,
-    supports_weights         = false,
-    docstring                = "log cosh loss; aliases: `log_cosh`.")
+    orientation              = :loss)
 
 const LogCosh = LogCoshLoss
 @create_aliases LogCoshLoss
@@ -341,7 +284,4 @@ body="Reports ``\\log(\\cosh(ŷᵢ-yᵢ))`` for each index `i`. ")
 _softplus(x::T) where T<:Real = x > zero(T) ? x + log1p(exp(-x)) : log1p(exp(x))
 _log_cosh(x::T) where T<:Real = x + _softplus(-2x) - log(convert(T, 2))
 
-function (log_cosh::LogCoshLoss)(ŷ::Vec{<:T}, y::Vec{<:T}) where T <:Real
-    check_dimensions(ŷ, y)
-    return _log_cosh.(ŷ - y)
-end
+single(::LogCoshLoss, ŷ::Real, y::Real) = _log_cosh(ŷ - y)
diff --git a/src/measures/finite.jl b/src/measures/finite.jl
index 681bf226..811fcac5 100644
--- a/src/measures/finite.jl
+++ b/src/measures/finite.jl
@@ -1,222 +1,17 @@
-# ============================================================
-# PROBABILISTIC PREDICTIONS
-
-# -----------------------------------------------------
-# LogLoss
-
-struct LogLoss{R <: Real} <: Measure
-    tol::R
-end
-LogLoss(;eps=eps(), tol=eps) = LogLoss(tol)
-
-metadata_measure(LogLoss;
-                 instances                = ["log_loss", "cross_entropy"],
-                 target_scitype           = Arr{<:Union{Missing,Finite}},
-                 prediction_type          = :probabilistic,
-                 orientation              = :loss,
-                 reports_each_observation = true,
-                 is_feature_dependent     = false,
-                 supports_weights         = false,
-                 distribution_type        = UnivariateFinite)
-
-const CrossEntropy = LogLoss
-@create_aliases LogLoss
-
-@create_docs(LogLoss,
-body=
-"""
-Since the score is undefined in the case that the true observation is
-predicted to occur with probability zero, probablities are clipped
-between `tol` and `1-tol`, where `tol` is a constructor key-word
-argument.
-
-If `sᵢ` is the predicted probability for the true class `yᵢ` then
-the score for that example is given by
-
-    -log(clamp(sᵢ, tol), 1 - tol)
-
-A score is reported for every observation.
-""",
-scitype=DOC_FINITE)
-
-# workaround for https://github.com/JuliaLang/julia/issues/41939:
-@static if VERSION < v"1.1"
-    Base.clamp(::Missing, lo::Any, hi::Any) = missing
-end
-
-# for single observation:
-_cross_entropy(d::UnivariateFinite{S,V,R,P}, y, tol) where {S,V,R,P} =
-    -log(clamp(pdf(d, y), P(tol), P(1) - P(tol)))
-
-# multiple observations:
-function (c::LogLoss)(ŷ::Arr{<:UnivariateFinite,N},
-                      y::Arr{V,N}) where {V,N}
-    check_dimensions(ŷ, y)
-#    check_pools(ŷ, y)
-    return broadcast(_cross_entropy, ŷ, y, c.tol)
-end
-# performant in case of UnivariateFiniteArray:
-function (c::LogLoss)(ŷ::UnivariateFiniteArray{S,V,R,P,N},
-                      y::ArrMissing{V,N}) where {S,V,R,P,N}
-    check_dimensions(ŷ, y)
-#    check_pools(ŷ, y)
-    return -log.(clamp.(broadcast(pdf, ŷ, y), P(c.tol), P(1) - P(c.tol)))
-end
-
-# -----------------------------------------------------
-# BrierScore
-
-struct BrierScore <: Measure end
-
-metadata_measure(BrierScore;
-                 human_name = "Brier score (a.k.a. quadratic score)",
-                 instances                = ["brier_score",],
-                 target_scitype           = Arr{<:Union{Missing,Finite}},
-                 prediction_type          = :probabilistic,
-                 orientation              = :score,
-                 reports_each_observation = true,
-                 is_feature_dependent     = false,
-                 supports_weights         = true,
-                 distribution_type        = UnivariateFinite)
-
-@create_aliases BrierScore
-
-@create_docs(BrierScore,
-body=
-"""
-If `p(y)` is the predicted probability for a *single*
-observation `y`, and `C` all possible classes, then the corresponding
-Brier score for that observation is given by
-
-``2p(y) - \\left(\\sum_{η ∈ C} p(η)^2\\right) - 1``
-
-*Warning.* `BrierScore()` is a "score" in the sense that bigger is
-better (with `0` optimal, and all other values negative). In Brier's
-original 1950 paper, and many other places, it has the opposite sign,
-despite the name. Moreover, the present implementation does not treat
-the binary case as special, so that the score may differ, in that
-case, by a factor of two from usage elsewhere.
-""",
-scitype=DOC_FINITE)
-
-# calling on single observations (no checks):
-function _brier_score(d::UnivariateFinite{S,V,R,P}, y) where {S,V,R,P}
-    levels = classes(d)
-    pvec = broadcast(pdf, d, levels)
-    offset = P(1) + sum(pvec.^2)
-    return P(2) * pdf(d, y) - offset
-end
-
-# calling on multiple observations:
-function (::BrierScore)(ŷ::Arr{<:UnivariateFinite,N},
-                        y::Arr{V,N},
-                        w::Union{Nothing,Arr{<:Real,N}}=nothing) where {V,N}
-    check_dimensions(ŷ, y)
-    w === nothing || check_dimensions(w, y)
-
-    check_pools(ŷ, y)
-    unweighted = broadcast(_brier_score, ŷ, y)
-
-    if w === nothing
-        return unweighted
-    end
-    return w.*unweighted
-end
-
-# Performant version in case of UnivariateFiniteArray:
-function (::BrierScore)(
-    ŷ::UnivariateFiniteArray{S,V,R,P,N},
-    y::ArrMissing{V,N},
-    w::Union{Nothing,Arr{<:Real,N}}=nothing) where {S,V,R,P<:Real,N}
-
-    check_dimensions(ŷ, y)
-    w === nothing || check_dimensions(w, y)
-
-    isempty(y) && return P(0)
-
-    check_pools(ŷ, y)
-
-    probs = pdf(ŷ, classes(first(ŷ)))
-    offset = P(1) .+ vec(sum(probs.^2, dims=2))
-
-    unweighted = P(2) .* broadcast(pdf, ŷ, y) .- offset
-
-    if w === nothing
-        return unweighted
-    end
-    return w.*unweighted
-end
-
-# -----------------------------------------------------
-# BrierLoss
-
-struct BrierLoss <: Measure end
-
-metadata_measure(BrierLoss;
-                 human_name = "Brier loss (a.k.a. quadratic loss)",
-                 instances                = ["brier_loss",],
-                 target_scitype           = Arr{<:Union{Missing,Finite}},
-                 prediction_type          = :probabilistic,
-                 orientation              = :score,
-                 reports_each_observation = true,
-                 is_feature_dependent     = false,
-                 supports_weights         = true,
-                 distribution_type        = UnivariateFinite)
-
-@create_aliases BrierLoss
-
-@create_docs(BrierLoss,
-body=
-"""
-If `p(y)` is the predicted probability for a *single*
-observation `y`, and `C` all possible classes, then the corresponding
-Brier score for that observation is given by
-
-``\\left(\\sum_{η ∈ C} p(η)^2\\right) - 2p(y) + 1``
-
-*Warning.* In Brier's original 1950 paper, what is implemented here is
-called a "loss". It is, however, a "score" in the contemporary use of
-that term: smaller is better (with `0` optimal, and all other values
-positive).  Note also the present implementation does not treat the
-binary case as special, so that the loss may differ, in that case, by
-a factor of two from usage elsewhere.
-""",
-scitype=DOC_FINITE)
-
-# calling on single observations (no checks):
-function _brier_loss(d::UnivariateFinite{S,V,R,P}, y) where {S,V,R,P}
-    levels = classes(d)
-    pvec = broadcast(pdf, d, levels)
-    offset = P(1) + sum(pvec.^2)
-    return P(2) * pdf(d, y) - offset
-end
-
-(m::BrierLoss)(ŷ::Arr{<:UnivariateFinite,N},
-               y::Arr{V,N},
-               w::Union{Nothing,Arr{<:Real,N}}=nothing) where {V,N} =
-                   - brier_score(ŷ, y, w)
-
-const INVARIANT_LABEL =
-    "This metric is invariant to class reordering."
-const VARIANT_LABEL =
-    "This metric is *not* invariant to class re-ordering"
-
-# =============================================================
-# DETERMINISTIC FINITE PREDICTIONS
+const FiniteArrMissing{N} = Union{
+    AbstractArray{<:Union{Missing,Multiclass{N}}},
+    AbstractArray{<:Union{Missing,OrderedFactor{N}}}}
 
 # ---------------------------------------------------
 # misclassification rate
 
-struct MisclassificationRate <: Measure end
+struct MisclassificationRate <: Aggregated end
 
 metadata_measure(MisclassificationRate;
                  instances  = ["misclassification_rate", "mcr"],
-                 target_scitype           = Arr{<:Union{Missing,Finite}},
+                 target_scitype           = FiniteArrMissing,
                  prediction_type          = :deterministic,
-                 orientation              = :loss,
-                 reports_each_observation = false,
-                 is_feature_dependent     = false,
-                 supports_weights         = true)
+                 orientation              = :loss)
 
 const MCR = MisclassificationRate
 @create_aliases MCR
@@ -230,25 +25,20 @@ $INVARIANT_LABEL
 scitype=DOC_FINITE)
 
 # calling behaviour:
-(::MCR)(ŷ, y) where {V,N} =
-    mean(skipmissing(y .!= ŷ))
-(::MCR)(ŷ, y, w) where {V,N} =
-    mean(skipmissing((y .!= ŷ) .* w))
+call(::MCR, ŷ, y) where {V,N} = (y .!= ŷ) |> Mean()
+call(::MCR, ŷ, y, w) where {V,N} = (y .!= ŷ) .* w |> Mean()
 (::MCR)(cm::ConfusionMatrixObject) = 1.0 - sum(diag(cm.mat)) / sum(cm.mat)
 
 # -------------------------------------------------------------
 # accuracy
 
-struct Accuracy <: Measure end
+struct Accuracy <: Aggregated end
 
 metadata_measure(Accuracy;
                  instances = ["accuracy",],
-                 target_scitype           = Arr{<:Union{Missing,Finite}},
+                 target_scitype           = FiniteArrMissing,
                  prediction_type          = :deterministic,
-                 orientation              = :score,
-                 reports_each_observation = false,
-                 is_feature_dependent     = false,
-                 supports_weights         = true),
+                 orientation              = :score)
 
 @create_aliases Accuracy
 
@@ -261,22 +51,19 @@ ground truth `y[i]` observations. $INVARIANT_LABEL
 scitype=DOC_FINITE)
 
 # calling behaviour:
-(::Accuracy)(args...) = 1.0 - misclassification_rate(args...)
+call(::Accuracy, args...) = 1.0 - call(misclassification_rate, args...)
 (::Accuracy)(m::ConfusionMatrixObject) = sum(diag(m.mat)) / sum(m.mat)
 
 # -----------------------------------------------------------
 # balanced accuracy
 
-struct BalancedAccuracy <: Measure end
+struct BalancedAccuracy <: Aggregated end
 
 metadata_measure(BalancedAccuracy;
                  instances = ["balanced_accuracy", "bacc", "bac"],
-                 target_scitype           = Arr{<:Union{Missing,Finite}},
+                 target_scitype           = FiniteArrMissing,
                  prediction_type          = :deterministic,
-                 orientation              = :score,
-                 reports_each_observation = false,
-                 is_feature_dependent     = false,
-                 supports_weights         = true)
+                 orientation              = :score)
 
 const BACC = BalancedAccuracy
 @create_aliases BACC
@@ -290,24 +77,23 @@ $INVARIANT_LABEL
 """,
 scitype=DOC_FINITE)
 
-# calling behavior:
+function call(::BACC, ŷm, ym, wm::Union{Nothing,Arr{<:Real}}=nothing)
 
-function (::BACC)(ŷ, y)
-    class_count = Dist.countmap(y)
-    ŵ = 1.0 ./ [class_count[yi] for yi in y]
-    weighted_matches = (ŷ .== y) .* ŵ
-    mask = .!(ismissing.(weighted_matches))
-    return sum(skipmissing(weighted_matches)) / sum(ŵ[mask])
-end
+    ŷ, y, w = _skipinvalid(ŷm, ym, wm)
 
-function (::BACC)(ŷm, ym, w)
-    ŷ, y, w = _skipmissing(ŷm, ym, w)
-    levels_ = levels(y)
-    ŵ = similar(w)
-    @inbounds for i in eachindex(w)
-        ŵ[i] = w[i] / sum(w .* (y .== y[i]))
+    if w == nothing
+        n_given_class = StatsBase.countmap(y)
+        freq(i) = @inbounds n_given_class[y[i]]
+        ŵ = 1 ./ freq.(eachindex(y))
+    else # following sklearn, which is non-linear
+        ŵ = similar(w)
+        @inbounds for i in eachindex(w)
+            ŵ[i] = w[i] / sum(w .* (y .== y[i]))
+        end
     end
-    return sum( (ŷ .== y) .* ŵ ) / sum(ŵ)
+    s = sum(ŵ)
+
+    return  sum((ŷ .== y) .* ŵ) / sum(ŵ)
 end
 
 
@@ -317,15 +103,13 @@ end
 # ------------------------------------------------------------------
 # Matthew's correlation
 
-struct MatthewsCorrelation <: Measure end
+struct MatthewsCorrelation <: Aggregated end
 
 metadata_measure(MatthewsCorrelation;
                  instances = ["matthews_correlation", "mcc"],
-                 target_scitype           = Arr{<:Union{Missing,Finite{2}}},
+                 target_scitype           = FiniteArrMissing{2},
                  prediction_type          = :deterministic,
                  orientation              = :score,
-                 reports_each_observation = false,
-                 is_feature_dependent     = false,
                  supports_weights         = false)
 const MCC = MatthewsCorrelation
 @create_aliases MCC
@@ -363,67 +147,7 @@ function (::MCC)(cm::ConfusionMatrixObject{C}) where C
     return mcc
 end
 
-(m::MCC)(ŷ, y) = _confmat(ŷ, y, warn=false) |> m
-
-# ---------------------------------------------------------
-# AreaUnderCurve
-
-#. Implementation drawn from
-# https://www.ibm.com/developerworks/community/blogs/jfp/entry/Fast_Computation_of_AUC_ROC_score?lang=en
-# but this link is now broken. Author contacted here:
-# https://www.kaggle.com/c/microsoft-malware-prediction/discussion/76013.
-
-struct AreaUnderCurve <: Measure end
-
-metadata_measure(AreaUnderCurve;
-                 human_name = "area under the ROC",
-                 instances = ["area_under_curve", "auc"],
-                 target_scitype           = Arr{<:Union{Missing,Finite{2}}},
-                 prediction_type          = :probabilistic,
-                 orientation              = :score,
-                 reports_each_observation = false,
-                 is_feature_dependent     = false,
-                 supports_weights         = false)
-
-const AUC = AreaUnderCurve
-@create_aliases AreaUnderCurve
-
-@create_docs(AreaUnderCurve,
-body=
-"""
-Returns the area under the ROC ([receiver operator
-characteristic](https://en.wikipedia.org/wiki/Receiver_operating_characteristic))
-$INVARIANT_LABEL
-""",
-scitpye = DOC_FINITE_BINARY)
-
-# core algorithm:
-function _auc(::Type{P}, ŷm, ym) where P<:Real # type of probabilities
-    ŷ, y    = _skipmissing(ŷm, ym)
-    lab_pos = classes(first(ŷ))[2] # 'positive' label
-    scores  = pdf.(ŷ, lab_pos)     # associated scores
-    y_sort  = y[sortperm(scores)]  # sort by scores
-    n       = length(y)
-    n_neg   = 0  # to keep of the number of negative preds
-    auc     = P(0)
-    @inbounds for i in 1:n
-        # y[i] == lab_p --> it's a positive label in the ground truth
-        # in that case increase the auc by the cumulative sum
-        # otherwise increase the number of negatives by 1
-        δ_auc, δ_neg = ifelse(y_sort[i] == lab_pos, (n_neg, 0), (0, 1))
-        auc   += δ_auc
-        n_neg += δ_neg
-    end
-    n_pos = n - n_neg
-    return auc / (n_neg * n_pos)
-end
-
-# calling behaviour:
-(::AUC)(ŷ::Arr{<:UnivariateFinite}, y) = _auc(Float64, ŷ, y)
-
-# performant version for UnivariateFiniteVector:
-(::AUC)(ŷ::Arr{<:UnivariateFinite{S,V,R,P}}, y) where {S,V,R,P} =
-    _auc(P, ŷ, y)
+call(m::MCC, ŷ, y) = _confmat(ŷ, y, warn=false) |> m
 
 
 # ==========================================================================
@@ -434,7 +158,7 @@ const CM2 = ConfusionMatrixObject{2}
 # --------------------------------------------------------------------------
 # FScore
 
-struct FScore{T<:Real} <: Measure
+struct FScore{T<:Real} <: Aggregated
     β::T
     rev::Union{Nothing,Bool}
 end
@@ -444,11 +168,9 @@ FScore(; β=1.0, rev=nothing) = FScore(β, rev)
 metadata_measure(FScore;
                  human_name = "F-Score",
                  instances = ["f1score",],
-                 target_scitype           = Arr{<:Union{Missing,Finite{2}}},
+                 target_scitype           = FiniteArrMissing{2},
                  prediction_type          = :deterministic,
                  orientation              = :score,
-                 reports_each_observation = false,
-                 is_feature_dependent     = false,
                  supports_weights         = false)
 
 @create_aliases FScore
@@ -479,7 +201,7 @@ function (score::FScore)(m::CM2)
 end
 
 # calling on arrays:
-(m::FScore)(ŷ, y) = _confmat(ŷ, y; rev=m.rev) |> m
+call(m::FScore, ŷ, y) = _confmat(ŷ, y; rev=m.rev) |> m
 
 # -------------------------------------------------------------------------
 # TruePositive and its cousins - struct and metadata declerations
@@ -492,45 +214,37 @@ const TRUE_POSITIVE_AND_COUSINS =
 
 for M in TRUE_POSITIVE_AND_COUSINS
     ex = quote
-        struct $M <: Measure rev::Union{Nothing,Bool} end
+        struct $M <: Aggregated rev::Union{Nothing,Bool} end
         $M(; rev=nothing) = $M(rev)
     end
     eval(ex)
 end
 
 metadata_measure.((FalsePositive, FalseNegative);
-    target_scitype           = Arr{<:Union{Missing,Finite{2}}},
+    target_scitype           = FiniteArrMissing{2},
     prediction_type          = :deterministic,
     orientation              = :loss,
-    reports_each_observation = false,
     aggregation              = Sum(),
-    is_feature_dependent     = false,
     supports_weights         = false)
 
 metadata_measure.((FalsePositiveRate, FalseNegativeRate, FalseDiscoveryRate);
-    target_scitype           = Arr{<:Union{Missing,Finite{2}}},
+    target_scitype           = FiniteArrMissing{2},
     prediction_type          = :deterministic,
     orientation              = :loss,
-    reports_each_observation = false,
-    is_feature_dependent     = false,
     supports_weights         = false)
 
 metadata_measure.((TruePositive, TrueNegative);
-    target_scitype           = Arr{<:Union{Missing,Finite{2}}},
+    target_scitype           = FiniteArrMissing{2},
     prediction_type          = :deterministic,
     orientation              = :score,
-    reports_each_observation = false,
     aggregation              = Sum(),
-    is_feature_dependent     = false,
     supports_weights         = false)
 
 metadata_measure.((TruePositiveRate, TrueNegativeRate, Precision,
                    NegativePredictiveValue);
-    target_scitype           = Arr{<:Union{Missing,Finite{2}}},
+    target_scitype           = FiniteArrMissing{2},
     prediction_type          = :deterministic,
     orientation              = :score,
-    reports_each_observation = false,
-    is_feature_dependent     = false,
     supports_weights         = false)
 
 # adjustments:
@@ -644,19 +358,22 @@ _npv(m::CM2) = _tn(m) / (_tn(m) + _fn(m))
 (::NPV)(m::CM2) = _npv(m)
 (::Precision)(m::CM2) = 1.0 - _fdr(m)
 
-# on vectors (ŷ, y):
+# on arrays (ŷ, y):
 for M_ex in TRUE_POSITIVE_AND_COUSINS
-    local M = eval(M_ex)
-    (m::M)(ŷ, y) = _confmat(ŷ, y; rev=m.rev) |> m
+    @eval call(m::$M_ex, ŷ, y) = _confmat(ŷ, y; rev=m.rev) |> m
 end
 
-# special `precision` case (conflict with Base.precision):
+# since Base.precision exists (as single argument function) we
+# manually overload Base.precision:
 Base.precision(m::CM2) = m |> Precision()
-Base.precision(ŷ, y)   = _confmat(ŷ, y) |> Precision()
+function Base.precision(ŷ, y)
+    _check(Precision(), ŷ, y)
+    call(Precision(), ŷ, y)
+end
 
 
 # =================================================================
-#MULTICLASS AND ORDER INDEPENDENT
+# MULTICLASS AND ORDER INDEPENDENT
 
 const CM = ConfusionMatrixObject{N} where N
 
@@ -699,7 +416,7 @@ For more information, run `info(MulticlassFScore)`.
 """
 struct MulticlassFScore{T<:Real,
                         M<:MulticlassAvg,
-                        U<:Union{Vector, LittleDict}} <:Measure
+                        U<:Union{Vector, LittleDict}} <:Aggregated
     β::T
     average::M
     return_type::Type{U}
@@ -711,11 +428,9 @@ MulticlassFScore(; β=1.0, average=macro_avg, return_type=LittleDict) =
 metadata_measure(MulticlassFScore;
                  instances = ["macro_f1score", "micro_f1score",
                               "multiclass_f1score"],
-                 target_scitype           = Arr{<:Union{Missing,Finite{N}}} where N,
+                 target_scitype           = FiniteArrMissing,
                  prediction_type          = :deterministic,
                  orientation              = :score,
-                 reports_each_observation = false,
-                 is_feature_dependent     = false,
                  supports_weights         = false,
                  supports_class_weights   = true)
 
@@ -732,7 +447,7 @@ const multiclass_f1score = MulticlassFScore(average=macro_avg)
 for M in (:MulticlassTruePositive, :MulticlassTrueNegative,
           :MulticlassFalsePositive, :MulticlassFalseNegative)
     ex = quote
-        struct $M{U<:Union{Vector, LittleDict}} <: Measure
+        struct $M{U<:Union{Vector, LittleDict}} <: Aggregated
             return_type::Type{U}
         end
 #        $M(return_type::Type{U}) where {U} = $M(return_type)
@@ -751,7 +466,7 @@ for M in (:MulticlassTruePositiveRate, :MulticlassTrueNegativeRate,
           :MulticlassFalseDiscoveryRate, :MulticlassPrecision,
           :MulticlassNegativePredictiveValue)
     ex = quote
-        struct $M{T<:MulticlassAvg, U<:Union{Vector, LittleDict}} <: Measure
+        struct $M{T<:MulticlassAvg, U<:Union{Vector, LittleDict}} <: Aggregated
             average::T
             return_type::Type{U}
         end
@@ -766,10 +481,9 @@ for M in (:MulticlassTruePositiveRate, :MulticlassTrueNegativeRate,
 end
 
 metadata_measure.((MulticlassFalsePositive, MulticlassFalseNegative);
-    target_scitype           = Arr{<:Union{Missing,Finite{N}}} where N,
+    target_scitype           = FiniteArrMissing,
     prediction_type          = :deterministic,
     orientation              = :loss,
-    reports_each_observation = false,
     aggregation               = Sum(),
     is_feature_dependent     = false,
     supports_weights         = false,
@@ -777,38 +491,34 @@ metadata_measure.((MulticlassFalsePositive, MulticlassFalseNegative);
 
 metadata_measure.((MulticlassFalsePositiveRate, MulticlassFalseNegativeRate,
                    MulticlassFalseDiscoveryRate);
-    target_scitype           = Arr{<:Union{Missing,Finite{N}}} where N,
+    target_scitype           = FiniteArrMissing,
     prediction_type          = :deterministic,
     orientation              = :loss,
-    reports_each_observation = false,
     is_feature_dependent     = false,
     supports_weights         = false,
     supports_class_weights   = true)
 
 metadata_measure.((MulticlassTruePositive, MulticlassTrueNegative);
-    target_scitype           = Arr{<:Union{Missing,Finite{N}}} where N,
+    target_scitype           = FiniteArrMissing,
     prediction_type          = :deterministic,
     orientation              = :score,
-    reports_each_observation = false,
     aggregation              = Sum(),
     is_feature_dependent     = false,
     supports_weights         = false,
     supports_class_weights   = false)
 
 metadata_measure.((MulticlassTrueNegativeRate, MulticlassNegativePredictiveValue);
-    target_scitype           = Arr{<:Union{Missing,Finite{N}}} where N,
+    target_scitype           = FiniteArrMissing,
     prediction_type          = :deterministic,
     orientation              = :score,
-    reports_each_observation = false,
     is_feature_dependent     = false,
     supports_weights         = false,
     supports_class_weights   = true)
 
 metadata_measure.((MulticlassTruePositiveRate, MulticlassPrecision);
-    target_scitype           = Arr{<:Union{Missing,Finite{N}}} where N,
+    target_scitype           = FiniteArrMissing,
     prediction_type          = :deterministic,
     orientation              = :score,
-    reports_each_observation = false,
     is_feature_dependent     = false,
     supports_weights         = false,
     supports_class_weights   = true)
@@ -1162,7 +872,7 @@ const MNPV                                 = MulticlassNegativePredictiveValue
 
 
 # -----------------------------------------------------
-## INTERNAL FUCNTIONS ON MULTICLASS CONFUSION MATRIX
+## INTERNAL FUNCTIONS ON MULTICLASS CONFUSION MATRIX
 
 _mtp(m::CM, return_type::Type{Vector}) = diag(m.mat)
 _mtp(m::CM, return_type::Type{LittleDict}) =
@@ -1410,9 +1120,8 @@ end
 
 (p::MulticlassPrecision)(m::CM) =
     _mc_helper_b(m, _mfdr, p.average, p.return_type)
-function (p::MulticlassPrecision)(m::CM, class_w::AbstractDict{<:Any, <:Real})
-    return _mc_helper_b(m, _mfdr, class_w, p.average, p.return_type)
-end
+(p::MulticlassPrecision)(m::CM, class_w::AbstractDict{<:Any, <:Real}) =
+    _mc_helper_b(m, _mfdr, class_w, p.average, p.return_type)
 
 @inline function _fs_helper(m::CM, β::Real, rec::Arr{<:Real}, prec::Arr{<:Real},
                     average::NoAvg, return_type::Type{LittleDict})
@@ -1480,103 +1189,16 @@ function (f::MulticlassFScore)(m::CM, class_w::AbstractDict{<:Any, <:Real})
     return _fs_helper(m, class_w, f.β, f.average, f.return_type)
 end
 
-## Callables on vectors
+## Callables on arrays
 
-for M in (MulticlassTruePositive, MulticlassTrueNegative,
-          MulticlassFalsePositive, MulticlassFalseNegative)
-    (m::M)(ŷ, y) = m(_confmat(ŷ, y, warn=false))
+for M_ex in (:MulticlassTruePositive, :MulticlassTrueNegative,
+          :MulticlassFalsePositive, :MulticlassFalseNegative)
+    @eval call(m::$M_ex, ŷ, y) = m(_confmat(ŷ, y, warn=false))
 end
 
-for M in (MTPR, MTNR, MFPR, MFNR, MFDR, MulticlassPrecision, MNPV,
-          MulticlassFScore)
-    @eval (m::$M)(ŷ, y) = m(_confmat(ŷ, y, warn=false))
-    @eval (m::$M)(ŷ, y, class_w::AbstractDict{<:Any, <:Real}) =
-                          m(_confmat(ŷ, y, warn=false), class_w)
+for M_ex in (:MTPR, :MTNR, :MFPR, :MFNR, :MFDR, :MulticlassPrecision, :MNPV,
+          :MulticlassFScore)
+    @eval call(m::$M_ex, ŷ, y) = m(_confmat(ŷ, y, warn=false))
+    @eval call(m::$M_ex, ŷ, y, class_w::AbstractDict{<:Any, <:Real}) =
+        m(_confmat(ŷ, y, warn=false), class_w)
 end
-
-
-## ROC COMPUTATION
-
-"""
-    _idx_unique_sorted(v)
-
-Internal function to return the index of unique elements in `v` under the
-assumption that the vector `v` is sorted in decreasing order.
-"""
-function _idx_unique_sorted(v::Vec{<:Real})
-    n    = length(v)
-    idx  = ones(Int, n)
-    p, h = 1, 1
-    cur  = v[1]
-    @inbounds while h < n
-        h     += 1                  # head position
-        cand   = v[h]               # candidate value
-        cand   < cur || continue    # is it new? otherwise skip
-        p     += 1                  # if new store it
-        idx[p] = h
-        cur    = cand               # and update the last seen value
-    end
-    p < n && deleteat!(idx, p+1:n)
-    return idx
-end
-
-"""
-    fprs, tprs, ts = roc_curve(ŷ, y) = roc(ŷ, y)
-
-Return the ROC curve for a two-class probabilistic prediction `ŷ` given the
-ground  truth `y`. The true positive rates, false positive rates over a range
-of thresholds `ts` are returned. Note that if there are `k` unique scores,
-there are correspondingly  `k` thresholds and `k+1` "bins" over which the FPR
-and TPR are constant:
-
-* `[0.0 - thresh[1]]`
-* `[thresh[1] - thresh[2]]`
-* ...
-* `[thresh[k] - 1]`
-
-consequently, `tprs` and `fprs` are of length `k+1` if `ts` is of length `k`.
-
-To draw the curve using your favorite plotting backend, do `plot(fprs, tprs)`.
-"""
-function roc_curve(ŷm, ym)
-    ŷ, y    = _skipmissing(ŷm, ym)
-    n       = length(y)
-    lab_pos = levels(y)[2]
-    scores  = pdf.(ŷ, lab_pos)
-    ranking = sortperm(scores, rev=true)
-
-    scores_sort = scores[ranking]
-    y_sort_bin  = (y[ranking] .== lab_pos)
-
-    idx_unique = _idx_unique_sorted(scores_sort)
-    thresholds = scores_sort[idx_unique]
-
-    # detailed computations with example:
-    # y = [  1   0   0   1   0   0   1]
-    # s = [0.5 0.5 0.2 0.2 0.1 0.1 0.1] thresh are 0.5 0.2, 0.1 // idx [1, 3, 5]
-    # ŷ = [  0   0   0   0   0   0   0] (0.5 - 1.0] # no pos pred
-    # ŷ = [  1   1   0   0   0   0   0] (0.2 - 0.5] # 2 pos pred
-    # ŷ = [  1   1   1   1   0   0   0] (0.1 - 0.2] # 4 pos pred
-    # ŷ = [  1   1   1   1   1   1   1] [0.0 - 0.1] # all pos pre
-
-    idx_unique_2 = idx_unique[2:end]   # [3, 5]
-    n_ŷ_pos      = idx_unique_2 .- 1   # [2, 4] implicit [0, 2, 4, 7]
-
-    cs   = cumsum(y_sort_bin)          # [1, 1, 1, 2, 2, 2, 3]
-    n_tp = cs[n_ŷ_pos]                 # [1, 2] implicit [0, 1, 2, 3]
-    n_fp = n_ŷ_pos .- n_tp             # [1, 2] implicit [0, 1, 2, 4]
-
-    # add end points
-    P = sum(y_sort_bin) # total number of true positives
-    N = n - P           # total number of true negatives
-
-    n_tp = [0, n_tp..., P] # [0, 1, 2, 3]
-    n_fp = [0, n_fp..., N] # [0, 1, 2, 4]
-
-    tprs = n_tp ./ P  # [0/3, 1/3, 2/3, 1]
-    fprs = n_fp ./ N  # [0/4, 1/4, 2/4, 1]
-
-    return fprs, tprs, thresholds
-end
-
-const roc = roc_curve
diff --git a/src/measures/loss_functions_interface.jl b/src/measures/loss_functions_interface.jl
index 2a9ae04c..ca2c8afa 100644
--- a/src/measures/loss_functions_interface.jl
+++ b/src/measures/loss_functions_interface.jl
@@ -7,7 +7,7 @@ const WITHOUT_PARAMETERS =
 
 ## WRAPPER
 
-abstract type SupervisedLoss <: Measure end
+abstract type SupervisedLoss <: Unaggregated end
 
 
 struct MarginLoss{L<:LossFunctions.MarginLoss} <: SupervisedLoss
@@ -87,14 +87,6 @@ macro wrap_loss(ex)
         throw(err_wrap(4))
     end
 
-    # define calling behaviour:
-    push!(program.args, quote
-          (m::$Loss_ex)(yhat, y) =
-              MLJBase.value(m, yhat, nothing, y, nothing)
-          (m::$Loss_ex)(yhat, y, w) =
-              MLJBase.value(m, yhat, nothing, y, w)
-          end)
-
     esc(program)
 end
 
@@ -123,23 +115,20 @@ docstring(M::Type{<:SupervisedLoss})       = name(M)
 instances(M::Type{<:SupervisedLoss}) = [snakecase(string.(naked(M))), ]
 
 
-## DISTANCE BASED LOSS FUNCTIONS
+## CALLING - DISTANCE BASED LOSS FUNCTIONS
 
 MMI.prediction_type(::Type{<:DistanceLoss}) = :deterministic
 MMI.target_scitype(::Type{<:DistanceLoss}) = Union{Vec{Continuous},Vec{Count}}
 
-function value(measure::DistanceLoss, yhat, X, y, ::Nothing,
-                ::Val{false}, ::Val{true})
-    return LossFunctions.value(getfield(measure, :loss), y, yhat)
-end
+call(measure::DistanceLoss, yhat, y) =
+    LossFunctions.value(getfield(measure, :loss), y, yhat)
 
-function value(measure::DistanceLoss, yhat, X, y, w,
-                ::Val{false}, ::Val{true})
-    return w .* value(measure, yhat, X, y, nothing) ./ (sum(w)/length(y))
+function call(measure::DistanceLoss, yhat, y, w::ArrMissing{Real})
+    return w .* call(measure, yhat, y)
 end
 
 
-## MARGIN BASED LOSS FUNCTIONS
+## CALLING - MARGIN BASED LOSS FUNCTIONS
 
 MMI.prediction_type(::Type{<:MarginLoss}) = :probabilistic
 MMI.target_scitype(::Type{<:MarginLoss})  = AbstractArray{<:Finite{2}}
@@ -147,18 +136,14 @@ MMI.target_scitype(::Type{<:MarginLoss})  = AbstractArray{<:Finite{2}}
 # rescale [0, 1] -> [-1, 1]:
 _scale(p) = 2p - 1
 
-function value(measure::MarginLoss, yhat, X, y, ::Nothing,
-                ::Val{false}, ::Val{true})
-    check_pools(yhat, y)
+function call(measure::MarginLoss, yhat, y)
     probs_of_observed = broadcast(pdf, yhat, y)
     return (LossFunctions.value).(getfield(measure, :loss),
                                   1, _scale.(probs_of_observed))
 end
 
-function value(measure::MarginLoss, yhat, X, y, w,
-                ::Val{false}, ::Val{true})
-    return w .* value(measure, yhat, X, y, nothing) ./ (sum(w)/length(y))
-end
+call(measure::MarginLoss, yhat, y, w::ArrMissing{Real}) =
+    w .* call(measure, yhat, y)
 
 
 ## ADJUSTMENTS
@@ -179,10 +164,14 @@ _signature(::Type{<:QuantileLoss}) = "`QuantileLoss(; τ=0.7)`"
 
 ## ALIASES AND DOCSTRINGS
 
-const DOC_LOSS_FUNCTIONS = "For more detail, see "*
-    "the original LossFunctions.jl "*
-    "documentation **but "*
-    "note differences in the signature**. "
+const DOC_LOSS_FUNCTIONS =
+"""
+For more detail, see the original LossFunctions.jl documentation *but
+note differences in the signature.
+
+Losses from LossFunctions.jl do not support `missing` values. To use
+with `missing` values, replace `(ŷ, y)` with `skipinvalid(ŷ, y))`.
+"""
 
 for Loss_ex in DISTANCE_LOSSES
     eval(quote
diff --git a/src/measures/measure_search.jl b/src/measures/measure_search.jl
index 17048bb5..7c985822 100644
--- a/src/measures/measure_search.jl
+++ b/src/measures/measure_search.jl
@@ -1,5 +1,6 @@
 const LOCAL_MEASURE_TYPES = filter(x->x != SupervisedLoss,
-                                   subtypes(MLJBase.Measure))
+                                   vcat(subtypes(MLJBase.Unaggregated),
+                                        subtypes(MLJBase.Aggregated)))
 
 const LOSS_FUNCTIONS_MEASURE_TYPES =
     [eval(:($Loss)) for Loss in LOSS_FUNCTIONS]
diff --git a/src/measures/measures.jl b/src/measures/measures.jl
index f295cb96..ce0d88bb 100644
--- a/src/measures/measures.jl
+++ b/src/measures/measures.jl
@@ -1,3 +1,6 @@
+const PROPER_SCORING_RULES = "[Gneiting and Raftery (2007), \"Strictly"*
+    "Proper Scoring Rules, Prediction, and Estimation\""*
+    "](https://doi.org/10.1198/016214506000001437)"
 const DOC_FINITE =
     "`AbstractArray{<:Finite}` (multiclass classification)"
 const DOC_FINITE_BINARY =
@@ -8,12 +11,16 @@ const DOC_ORDERED_FACTOR_BINARY =
     "`AbstractArray{<:OrderedFactor{2}}` "*
     "(binary classification where choice of \"true\" effects the measure)"
 const DOC_CONTINUOUS = "`AbstractArray{Continuous}` (regression)"
-
-## TRAITS
+const DOC_COUNT = "`AbstractArray{Count}`"
+const DOC_INFINITE = "AbstractArray{<:Infinite}"
+const INVARIANT_LABEL =
+    "This metric is invariant to class reordering."
+const VARIANT_LABEL =
+    "This metric is *not* invariant to class re-ordering"
 
 is_measure_type(::Any) = false
 
-# The following traits, with fallbacks defined in
+# Each of the following traits, with fallbacks defined in
 # StatisticalTraits.jl, make sense for some or all measures:
 
 const MEASURE_TRAITS = [:name,
@@ -30,9 +37,93 @@ const MEASURE_TRAITS = [:name,
                         :docstring,
                         :distribution_type]
 
-## FOR BUILT-IN MEASURES (subtyping Measure)
+# # FOR BUILT-IN MEASURES (subtyping Measure)
 
 abstract type Measure <: MLJType end
+abstract type Aggregated <: Measure end
+abstract type Unaggregated <: Measure end
+
+StatisticalTraits.reports_each_observation(::Type{<:Aggregated}) = false
+StatisticalTraits.reports_each_observation(::Type{<:Unaggregated}) = true
+
+
+# # FALLBACK CHECKS
+
+extra_check(::Measure, args...) = nothing
+function _check(measure::Measure, yhat, y)
+    check_dimensions(yhat, y)
+    extra_check(measure, yhat, y)
+end
+function _check(measure::Measure, yhat, y, w)
+    check_dimensions(yhat, y)
+    extra_check(measure, yhat, y, w)
+end
+function _check(measure::Measure, yhat, y, w::Arr)
+    check_dimensions(yhat, y)
+    check_dimensions(y, w)
+    extra_check(measure, yhat, y, w)
+end
+function _check(measure::Measure, yhat::Arr{<:UnivariateFinite})
+    check_dimensions(yhat, y)
+    check_pools(yhat, y)
+    extra_check(measure, yhat, y)
+end
+function _check(measure::Measure,
+                yhat::Arr{<:UnivariateFinite},
+                y,
+                w::Arr)
+    check_dimensions(yhat, y)
+    check_pools(yhat, y)
+    extra_check(measure, yhat, y, w)
+end
+function _check(measure::Measure,
+                yhat::Arr{<:UnivariateFinite},
+                y,
+                w::AbstractDict)
+    check_dimensions(yhat, y)
+    check_pools(yhat, y)
+    check_pools(yhat, w)
+    extra_check(measure, yhat, y, w)
+end
+
+
+# # METHODS TO EVALUATE MEASURES
+
+# See measures/README.md for details
+
+single(::Unaggregated, η̂::Missing, η)          = missing
+single(::Unaggregated, η̂,          η::Missing) = missing
+
+const Label = Union{CategoricalValue,Number,AbstractString,Symbol,AbstractChar}
+
+# closure for broadcasting:
+single(measure::Measure) = (ηhat, η) -> single(measure, ηhat, η)
+
+call(measure::Unaggregated, yhat, y) = broadcast(single(measure), yhat, y)
+function call(measure::Unaggregated, yhat, y, w::Arr)
+    unweighted = broadcast(single(measure), yhat, y) # `single` closure below
+    return w .* unweighted
+end
+function call(measure::Unaggregated, yhat, y, weight_given_class::AbstractDict)
+    unweighted = broadcast(single(measure), yhat, y) # `single` closure below
+    w = @inbounds broadcast(η -> weight_given_class[η], y)
+    return w .* unweighted
+end
+
+# ## Top level
+
+function (measure::Measure)(args...)
+    _check(measure, args...)
+    call(measure, args...)
+end
+
+# # TRAITS
+
+# user-bespoke measures will subtype `Measure` directly and the
+# following will therefore not apply:
+StatisticalTraits.supports_weights(::Type{<:Union{Aggregated,Unaggregated}}) =
+    true
+
 is_measure_type(::Type{<:Measure}) = true
 is_measure(m) = is_measure_type(typeof(m))
 
@@ -60,15 +151,15 @@ end
 ScientificTypes.info(m, ::Val{:measure}) = info(typeof(m))
 
 
-## AGGREGATION
+# # AGGREGATION
 
-(::Sum)(v) = sum(skipmissing(v))
+(::Sum)(v) = sum(skipinvalid(v))
 (::Sum)(v::LittleDict) = sum(values(v))
 
-(::Mean)(v) = mean(skipmissing(v))
+(::Mean)(v) = mean(skipinvalid(v))
 (::Mean)(v::LittleDict) = mean(values(v))
 
-(::RootMeanSquare)(v) = sqrt(mean(skipmissing(v).^2))
+(::RootMeanSquare)(v) = sqrt(mean(skipinvalid(v).^2))
 
 aggregate(v, measure) = aggregation(measure)(v)
 
@@ -77,7 +168,7 @@ const MeasureValue = Union{Real,Tuple{<:Real,<:Real}} # number or interval
 aggregate(x::MeasureValue, measure) = x
 
 
-## DISPATCH FOR EVALUATION
+# # UNIVERSAL CALLING SYNTAX
 
 # yhat - predictions (point or probabilisitic)
 # X - features
@@ -92,7 +183,7 @@ function value(measure, yhat, X, y, w)
 end
 
 
-## DEFAULT EVALUATION INTERFACE
+# # UNIVERSAL CALLING INTERFACE
 
 #  is feature independent, weights not supported:
 value(m, yhat, X, y, w, ::Val{false}, ::Val{false}) = m(yhat, y)
@@ -108,7 +199,10 @@ value(m, yhat, X, y, ::Nothing, ::Val{false}, ::Val{true}) = m(yhat, y)
 value(m, yhat, X, y, w,         ::Val{true}, ::Val{true}) = m(yhat, X, y, w)
 value(m, yhat, X, y, ::Nothing, ::Val{true}, ::Val{true}) = m(yhat, X, y)
 
-## helpers
+# # helpers
+
+_scale(x, w::Arr, i) = x*w[i]
+_scale(x, ::Nothing, i::Any) = x
 
 function check_pools(ŷ, y)
     levels(y) == levels(ŷ[1]) ||
@@ -117,42 +211,50 @@ function check_pools(ŷ, y)
     return nothing
 end
 
-function _skipmissing(yhat, y)
-    mask = .!(ismissing.(yhat) .| ismissing.(y))
-    return yhat[mask], y[mask]
+function check_pools(ŷ, w::AbstractDict)
+    Set(levels(ŷ[1])) == Set(keys(w)) ||
+        error("Conflicting categorical pools found "*
+              "in class weights and predictions. ")
+    return nothing
 end
 
-function _skipmissing(yhat, y, w)
-    mask = .!(ismissing.(yhat) .| ismissing.(y))
-    return yhat[mask], y[mask], w[mask]
-end
 
-## INCLUDE SPECIFIC MEASURES AND TOOLS
+# # INCLUDE SPECIFIC MEASURES AND TOOLS
 
 include("meta_utilities.jl")
-include("continuous.jl")
+include("roc.jl")
 include("confusion_matrix.jl")
+include("continuous.jl")
 include("finite.jl")
+include("probabilistic.jl")
 include("loss_functions_interface.jl")
 
-## DEFAULT MEASURES
+
+# # DEFAULT MEASURES
+
 default_measure(T, S) = nothing
 
 # Deterministic + Continuous / Count ==> RMS
 default_measure(::Type{<:Deterministic},
-                ::Type{<:Union{Vec{<:Continuous}, Vec{<:Count}}}) = rms
+                ::Type{<:Union{Vec{<:Union{Missing,Continuous}},
+                               Vec{<:Union{Missing,Count}}}}) = rms
 
 # Deterministic + Finite ==> Misclassification rate
 default_measure(::Type{<:Deterministic},
-                ::Type{<:Vec{<:Finite}}) = misclassification_rate
+                ::Type{<:Vec{<:Union{Missing,Finite}}}) = misclassification_rate
+
+# Probabilistic + Finite ==> log loss
+default_measure(::Type{<:Probabilistic},
+                ::Type{<:Vec{<:Union{Missing,Finite}}}) = log_loss
+
+# Probablistic + Continuous ==> Log loss
+default_measure(::Type{<:Probabilistic},
+                ::Type{<:Vec{<:Union{Missing,Continuous}}}) = log_loss
 
-# default_measure(::Type{Probabilistic},
-#                 ::Type{<:Union{Vec{<:Continuous},
-#                                Vec{<:Count}}}) = ???
 
-# Probabilistic + Finite ==> Cross entropy
+# Probablistic + Count ==> Log score
 default_measure(::Type{<:Probabilistic},
-                ::Type{<:Vec{<:Finite}}) = cross_entropy
+                ::Type{<:Vec{<:Union{Missing,Count}}}) = log_loss
 
 # Fallbacks
 default_measure(M::Type{<:Supervised}) = default_measure(M, target_scitype(M))
diff --git a/src/measures/meta_utilities.jl b/src/measures/meta_utilities.jl
index ef3ffa7c..e38425b0 100644
--- a/src/measures/meta_utilities.jl
+++ b/src/measures/meta_utilities.jl
@@ -1,6 +1,6 @@
 const DOC_OBSERVATIONS =
-    "on observations `ŷ`, "*
-    "given ground truth values `y`. "
+    "on predictions `ŷ`, "*
+    "given ground truth observations `y`. "
 const DOC_WEIGHTS =
     "Optionally specify per-sample weights, `w`. "
 const DOC_CLASS_WEIGHTS =
@@ -55,7 +55,7 @@ function detailed_doc_string(M; typename="", body="", footer="", scitype="")
     ret *= "\n\n"
     isempty(body) || (ret *= "$body\n\n")
     ret *= "Requires `scitype(y)` to be a subtype of $scitype; "
-    ret *= "`ŷ` must be a $(prediction_type(M)) prediction. "
+    ret *= "`ŷ` must be an array of $(prediction_type(M)) predictions. "
     isempty(footer) ||(ret *= "\n\n$footer")
     ret *= "\n\n"
     ret *= "For more information, run `info($(name(M)))`. "
@@ -98,13 +98,12 @@ function metadata_measure(T; name::String="",
                           target_scitype=Unknown,
                           prediction_type::Symbol=:unknown,
                           orientation::Symbol=:unknown,
-                          reports_each_observation::Bool=true,
                           aggregation=Mean(),
                           is_feature_dependent::Bool=false,
-                          supports_weights::Bool=false,
+                          supports_weights::Bool=true,
                           supports_class_weights::Bool=false,
                           docstring::String="",
-                          distribution_type=missing)
+                          distribution_type=Unknown)
     pred_str        = "$prediction_type"
     orientation_str = "$orientation"
 #    dist = ifelse(ismissing(distribution_type), missing, "$distribution_type")
@@ -112,28 +111,29 @@ function metadata_measure(T; name::String="",
 
         # traits common with models:
         if !isempty($name)
-            MMI.name(::Type{<:$T}) = $name
+            StatisticalTraits.name(::Type{<:$T}) = $name
         end
         if !isempty($docstring)
-            MMI.docstring(::Type{<:$T}) = $docstring
+            StatisticalTraits.docstring(::Type{<:$T}) = $docstring
         end
-        MMI.target_scitype(::Type{<:$T}) = $target_scitype
-        MMI.prediction_type(::Type{<:$T}) = Symbol($pred_str)
-        MMI.supports_weights(::Type{<:$T}) = $supports_weights
+        StatisticalTraits.target_scitype(::Type{<:$T}) = $target_scitype
+        StatisticalTraits.prediction_type(::Type{<:$T}) = Symbol($pred_str)
+        StatisticalTraits.supports_weights(::Type{<:$T}) = $supports_weights
 
         # traits specific to measures:
         if !isempty($instances)
-            instances(::Type{<:$T}) = $instances
+            StatisticalTraits.instances(::Type{<:$T}) = $instances
         end
         if !isempty($human_name)
-            human_name(::Type{<:$T}) = $human_name
+            StatisticalTraits.human_name(::Type{<:$T}) = $human_name
         end
-        orientation(::Type{<:$T}) = Symbol($orientation_str)
-        reports_each_observation(::Type{<:$T}) = $reports_each_observation
-        aggregation(::Type{<:$T}) = $aggregation
-        is_feature_dependent(::Type{<:$T}) = $is_feature_dependent
-        supports_class_weights(::Type{<:$T}) = $supports_class_weights
-        distribution_type(::Type{<:$T}) = $distribution_type
+        StatisticalTraits.orientation(::Type{<:$T}) = Symbol($orientation_str)
+        StatisticalTraits.aggregation(::Type{<:$T}) = $aggregation
+        StatisticalTraits.is_feature_dependent(::Type{<:$T}) =
+            $is_feature_dependent
+        StatisticalTraits.supports_class_weights(::Type{<:$T}) =
+            $supports_class_weights
+        StatisticalTraits.distribution_type(::Type{<:$T}) = $distribution_type
 
     end
     parentmodule(T).eval(ex)
diff --git a/src/measures/probabilistic.jl b/src/measures/probabilistic.jl
new file mode 100644
index 00000000..7e651c42
--- /dev/null
+++ b/src/measures/probabilistic.jl
@@ -0,0 +1,442 @@
+const DOC_DISTRIBUTIONS =
+"""
+In the case the predictions `ŷ` are continuous probability
+distributions, such as `Distributions.Normal`, replace the above sum
+with an integral, and interpret `p` as the probablity density
+function. In case of discrete distributions over the integers, such as
+`Distributions.Poisson`, sum over all integers instead of `C`.
+"""
+const WITH_L2NORM_CONTINUOUS =
+    [@eval(Distributions.$d) for d in [
+        :Chisq,
+        :Gamma,
+        :Beta,
+        :Chi,
+        :Cauchy,
+        :Normal,
+        :Uniform,
+        :Logistic,
+        :Exponential]]
+
+const WITH_L2NORM_COUNT =
+    [@eval(Distributions.$d) for d in [
+        :Poisson,
+        :DiscreteUniform,
+        :DiscreteNonParametric]]
+
+const WITH_L2NORM = vcat([UnivariateFinite, ],
+                                  WITH_L2NORM_CONTINUOUS,
+                         WITH_L2NORM_COUNT)
+
+const UD = Distributions.UnivariateDistribution
+
+# ========================================================
+# AGGREGATED MEASURES
+
+# ---------------------------------------------------------
+# AreaUnderCurve
+
+# Implementation drawn from
+# https://www.kaggle.com/c/microsoft-malware-prediction/discussion/76013.
+
+struct AreaUnderCurve <: Aggregated end
+
+metadata_measure(AreaUnderCurve;
+                 human_name = "area under the ROC",
+                 instances = ["area_under_curve", "auc"],
+                 target_scitype           = FiniteArrMissing{2},
+                 prediction_type          = :probabilistic,
+                 orientation              = :score,
+                 supports_weights         = false,
+                 distribution_type        = UnivariateFinite)
+
+const AUC = AreaUnderCurve
+@create_aliases AreaUnderCurve
+
+@create_docs(AreaUnderCurve,
+body=
+"""
+Returns the area under the ROC ([receiver operator
+characteristic](https://en.wikipedia.org/wiki/Receiver_operating_characteristic))
+
+If `missing` or `NaN` values are present, use `auc(skipinvalid(yhat, y)...)`.
+
+$INVARIANT_LABEL
+""",
+scitpye = DOC_FINITE_BINARY)
+
+# core algorithm:
+function _auc(::Type{P}, ŷ, y) where P<:Real # type of probabilities
+    lab_pos = classes(ŷ)[2] # 'positive' label
+    scores  = pdf.(ŷ, lab_pos)     # associated scores
+    y_sort  = y[sortperm(scores)]  # sort by scores
+    n       = length(y)
+    n_neg   = 0  # to keep of the number of negative preds
+    auc     = P(0)
+    @inbounds for i in 1:n
+        # y[i] == lab_p --> it's a positive label in the ground truth
+        # in that case increase the auc by the cumulative sum
+        # otherwise increase the number of negatives by 1
+        δ_auc, δ_neg = ifelse(y_sort[i] == lab_pos, (n_neg, 0), (0, 1))
+        auc   += δ_auc
+        n_neg += δ_neg
+    end
+    n_pos = n - n_neg
+    return auc / (n_neg * n_pos)
+end
+
+# Missing values not supported, but allow `Missing` in eltype, because
+# `skipinvalid(yhat, y)` does not tighten the type. See doc string above.
+
+call(::AUC, ŷ::ArrMissing{UnivariateFinite{S,V,R,P}}, y) where {S,V,R,P} =
+    _auc(P, ŷ, y)
+
+# corner case of UnivariateFinite's of mixed type
+call(::AUC, ŷ::ArrMissing{UnivariateFinite}, y) where {S,V,R,P} =
+    _auc(Float64, ŷ, y)
+
+
+
+# ========================================================
+# UNAGGREGATED MEASURES
+
+# ---------------------------------------------------------------------
+# LogScore
+
+struct LogScore{R <: Real} <: Unaggregated
+    tol::R
+end
+LogScore(;eps=eps(), tol=eps) = LogScore(tol)
+
+metadata_measure(LogScore;
+                 instances                = ["log_score", ],
+                 target_scitype           = Union{
+                     Arr{<:Union{Missing,Multiclass}},
+                     Arr{<:Union{Missing,OrderedFactor}},
+                     Arr{<:Union{Missing,Continuous}},
+                     Arr{<:Union{Missing,Count}}},
+                 prediction_type          = :probabilistic,
+                 orientation              = :score,
+                 distribution_type        = Union{WITH_L2NORM...})
+
+@create_aliases LogScore
+
+@create_docs(LogScore,
+body=
+"""
+Since the score is undefined in the case that the true observation is
+predicted to occur with probability zero, probablities are clamped
+between `tol` and `1-tol`, where `tol` is a constructor key-word
+argument.
+
+If `p` is the predicted probability mass or density function
+corresponding to a *single* ground truth observation `η`, then the
+score for that example is
+
+    log(clamp(p(η), tol), 1 - tol)
+
+For example, for a binary target with "yes"/"no" labels, and
+predicted probability of "yes" equal to 0.8, an observation of "no"
+scores `log(0.2)`.
+
+The predictions `ŷ` should be an array of `UnivariateFinite`
+distributions in the case of `Finite` target `y`, and otherwise a
+supported `Distributions.UnivariateDistribution` such as `Normal` or
+`Poisson`.
+
+See also [`LogLoss`](@ref), which differs only in sign.
+""")
+
+# for single finite observation:
+single(c::LogScore, d::UnivariateFinite{S,V,R,P}, η::Label) where {S,V,R,P} =
+    log(clamp(pdf(d, η), P(c.tol), P(1) - P(c.tol)))
+
+# for a single infinite observation:
+single(c::LogScore, d::Distributions.UnivariateDistribution, η::Real) =
+    log(clamp(pdf(d, η), c.tol, 1 - c.tol))
+
+# performant broadasting in case of UnivariateFiniteArray:
+function call(c::LogScore,
+              ŷ::UnivariateFiniteArray{S,V,R,P,N},
+              y::ArrMissing{V,N},
+              w::Union{Nothing,Arr{<:Real,N}}=nothing) where {S,V,R,P<:Real,N}
+    unweighted = log.(clamp.(broadcast(pdf, ŷ, y), P(c.tol), P(1) - P(c.tol)))
+    if w === nothing
+        return unweighted
+    end
+    return w .* unweighted
+end
+
+# ---------------------------------------------------------------------
+# LogLoss
+
+struct LogLoss{R <: Real} <: Unaggregated
+    tol::R
+end
+LogLoss(;eps=eps(), tol=eps) = LogLoss(tol)
+
+metadata_measure(LogLoss;
+                 instances                = ["log_loss", "cross_entropy"],
+                 target_scitype           = Union{
+                     Arr{<:Union{Missing,Multiclass}},
+                     Arr{<:Union{Missing,OrderedFactor}},
+                     Arr{<:Union{Missing,Continuous}},
+                     Arr{<:Union{Missing,Count}}},
+                 prediction_type          = :probabilistic,
+                 orientation              = :loss,
+                 distribution_type        = Union{WITH_L2NORM...})
+
+const CrossEntropy = LogLoss
+@create_aliases LogLoss
+
+@create_docs(LogLoss,
+body=
+"""
+For details, see [`LogScore`](@ref), which differs only by a sign.
+""")
+
+# for single finite observation:
+single(c::LogLoss, d::UnivariateFinite{S,V,R,P}, η::Label) where {S,V,R,P} =
+    -single(LogScore(tol=c.tol), d, η)
+
+# for a single infinite observation:
+single(c::LogLoss, d::Distributions.UnivariateDistribution, η::Real) =
+    -single(LogScore(tol=c.tol), d, η)
+
+# performant broadasting in case of UnivariateFiniteArray:
+call(c::LogLoss,
+     ŷ::UnivariateFiniteArray{S,V,R,P,N},
+     y::ArrMissing{V,N},
+     w::Union{Nothing,Arr{<:Real,N}}=nothing) where {S,V,R,P<:Real,N} =
+    -call(LogScore(tol=c.tol), ŷ, y, w)
+
+
+# -----------------------------------------------------
+# BrierScore
+
+struct BrierScore <: Unaggregated end
+
+metadata_measure(BrierScore;
+                 human_name = "Brier score (a.k.a. quadratic score)",
+                 instances                = ["brier_score",],
+                 target_scitype           = Union{
+                     Arr{<:Union{Missing,Multiclass}},
+                     Arr{<:Union{Missing,OrderedFactor}},
+                     Arr{<:Union{Missing,Continuous}},
+                     Arr{<:Union{Missing,Count}}},
+                 prediction_type          = :probabilistic,
+                 orientation              = :score,
+                 distribution_type        = Union{WITH_L2NORM...})
+
+@create_aliases BrierScore
+
+@create_docs(BrierScore,
+body=
+"""
+Convention as in $PROPER_SCORING_RULES
+
+*Finite case.* If `p` is the predicted probability mass function for a
+*single* observation `η`, and `C` all possible classes, then the
+corresponding score for that observation is given by
+
+``2p(η) - \\left(\\sum_{c ∈ C} p(c)^2\\right) - 1``
+
+*Warning.* `BrierScore()` is a "score" in the sense that bigger is
+better (with `0` optimal, and all other values negative). In Brier's
+original 1950 paper, and many other places, it has the opposite sign,
+despite the name. Moreover, the present implementation does not treat
+the binary case as special, so that the score may differ in the binary
+case by a factor of two from usage elsewhere.
+
+*Infinite case.* Replacing the sum above with an integral does *not*
+lead to the formula adopted here in the case of `Continuous` or
+`Count` target `y`. Rather the convention in the paper cited above is
+adopted, which means returning a score of
+
+``2p(η) - ∫ p(t)^2 dt``
+
+in the `Continuous` case (`p` the probablity density function) or
+
+``2p(η) - ∑_t p(t)^2``
+
+in the `Count` cae (`p` the probablity mass function).
+""",
+scitype=DOC_FINITE)
+
+# calling on single finite observation:
+function single(::BrierScore,
+                d::UnivariateFinite{S,V,R,P},
+                η::Label) where {S,V,R,P}
+    levels = classes(d)
+    pvec = broadcast(pdf, d, levels)
+    offset = P(1) + sum(pvec.^2)
+    return P(2) * pdf(d, η) - offset
+end
+
+# calling on a single infinite observation:
+single(::BrierScore, d::Distributions.UnivariateDistribution, η::Real) =
+    2*pdf(d, η) - Distributions.pdfsquaredL2norm(d)
+
+# Performant broadcasted version in case of UnivariateFiniteArray:
+function call(::BrierScore,
+              ŷ::UnivariateFiniteArray{S,V,R,P,N},
+              y::ArrMissing{V,N},
+              w::Union{Nothing,Arr{<:Real,N}}=nothing) where {S,V,R,P<:Real,N}
+
+    probs = pdf(ŷ, classes(first(ŷ)))
+    offset = P(1) .+ vec(sum(probs.^2, dims=2))
+
+    unweighted = P(2) .* broadcast(pdf, ŷ, y) .- offset
+
+    if w === nothing
+        return unweighted
+    end
+    return w.*unweighted
+end
+
+# -----------------------------------------------------
+# BrierLoss
+
+struct BrierLoss <: Unaggregated end
+
+metadata_measure(BrierLoss;
+                 human_name = "Brier loss (a.k.a. quadratic loss)",
+                 instances                = ["brier_loss",],
+                 target_scitype           = Union{
+                     Arr{<:Union{Missing,Multiclass}},
+                     Arr{<:Union{Missing,OrderedFactor}},
+                     Arr{<:Union{Missing,Continuous}},
+                     Arr{<:Union{Missing,Count}}},
+                 prediction_type          = :probabilistic,
+                 orientation              = :loss,
+                 distribution_type        = Union{WITH_L2NORM...})
+
+@create_aliases BrierLoss
+
+@create_docs(BrierLoss,
+body=
+"""
+For details, see [`BrierScore`](@ref), which differs only by a sign.
+""")
+
+# calling on single finite observations:
+single(::BrierLoss, d::UnivariateFinite{S,V,R,P}, η::Label) where {S,V,R,P} =
+    - single(BrierScore(), d, η)
+
+# calling on single infinite observations:
+single(::BrierLoss, d::Distributions.UnivariateDistribution, η::Real) =
+    -single(BrierScore(), d, η)
+
+# to get performant broadcasting in case of UnivariateFiniteArray
+call(m::BrierLoss,
+     ŷ::UnivariateFiniteArray{S,V,R,P,N},
+     y::ArrMissing{V,N},
+     w::Union{Nothing,Arr{<:Real,N}}=nothing) where {S,V,R,P<:Real,N} =
+         - call(BrierScore(), ŷ, y, w)
+
+# -----------------------------------------------------
+# SphericalScore
+
+struct SphericalScore{T<:Real} <: Unaggregated
+    alpha::T
+end
+SphericalScore(; alpha=2) = SphericalScore(alpha)
+
+metadata_measure(SphericalScore;
+                 human_name               = "Spherical score",
+                 instances                = ["spherical_score",],
+                 target_scitype           = Union{
+                     Arr{<:Union{Missing,Multiclass}},
+                     Arr{<:Union{Missing,OrderedFactor}},
+                     Arr{<:Union{Missing,Continuous}},
+                     Arr{<:Union{Missing,Count}}},
+                 prediction_type          = :probabilistic,
+                 orientation              = :score,
+                 distribution_type        = Union{WITH_L2NORM...})
+
+@create_aliases SphericalScore
+
+@create_docs(SphericalScore,
+body=
+"""
+Convention as in $PROPER_SCORING_RULES: If `η` takes on a finite
+number of classes `C` and ``p(η)` is the predicted probability for a
+*single* observation `η`, then the corresponding score for that
+observation is given by
+
+``p(y)^α / \\left(\\sum_{η ∈ C} p(η)^α\\right)^{1-α} - 1``
+
+where `α` is the measure parameter `alpha`.
+
+$DOC_DISTRIBUTIONS
+
+""")
+
+# calling on single observations:
+function single(s::SphericalScore,
+                d::UnivariateFinite{S,V,R,P}, η::Label) where {S,V,R,P}
+    α = s.alpha
+    levels = classes(d)
+    pvec = broadcast(pdf, d, levels)
+    return (pdf(d, η)/norm(pvec, α))^(α - 1)
+end
+
+single(s::SphericalScore, d::Distributions.UnivariateDistribution, η::Real) =
+    pdf(d, η)/sqrt(Distributions.pdfsquaredL2norm(d))
+
+# to compute the α-norm along last dimension:
+_norm(A::AbstractArray{<:Any,N}, α) where N =
+    sum(x -> x^α, A, dims=N)^(1/α)
+
+# Performant version in case of UnivariateFiniteArray:
+function call(s::SphericalScore,
+              ŷ::UnivariateFiniteArray{S,V,R,P,N},
+              y::ArrMissing{V,N},
+              w::Union{Nothing,Arr{<:Real,N}}=nothing) where {S,V,R,P<:Real,N}
+
+    α = s.alpha
+    alphanorm(A) = _norm(A, α)
+
+    predicted_probs = pdf(ŷ, classes(first(ŷ)))
+
+    unweighted = (broadcast(pdf, ŷ, y) ./ alphanorm(predicted_probs)).^(α - 1)
+
+    if w === nothing
+        return unweighted
+    end
+    return w.*unweighted
+end
+
+
+# ---------------------------------------------------------------------------
+# Extra check for L2 norm based proper scoring rules
+
+err_l2_norm(m) = ArgumentError(
+    "Distribution not supported by $m. "*
+    "Supported distributions are "*
+    join(string.(map(s->"`$s`", WITH_L2NORM)), ", ", ", and "))
+
+const ERR_UNSUPPORTED_ALPHA = ArgumentError(
+    "Only `alpha = 2` is supported, unless scoring a `Finite` target. ")
+
+# not for export:
+const L2ProperScoringRules = Union{LogScore,
+                                   LogLoss,
+                                   BrierScore,
+                                   BrierLoss,
+                                   SphericalScore}
+
+function extra_check(measure::L2ProperScoringRules, yhat, args...)
+
+    D = nonmissing(eltype(yhat))
+    D <: Distributions.Distribution ||
+        (D = typeof(findfirst(x->!isinvalid(x), yhat)))
+    D <: Union{Nothing, WITH_L2NORM...} ||
+        throw(err_l2_norm(measure))
+
+    if measure isa SphericalScore
+        measure.alpha == 2 || throw(ERR_UNSUPPORTED_ALPHA)
+    end
+
+    return nothing
+end
diff --git a/src/measures/roc.jl b/src/measures/roc.jl
new file mode 100644
index 00000000..aa028fb2
--- /dev/null
+++ b/src/measures/roc.jl
@@ -0,0 +1,85 @@
+## ROC COMPUTATION
+
+"""
+    _idx_unique_sorted(v)
+
+Internal function to return the index of unique elements in `v` under the
+assumption that the vector `v` is sorted in decreasing order.
+"""
+function _idx_unique_sorted(v::Vec{<:Real})
+    n    = length(v)
+    idx  = ones(Int, n)
+    p, h = 1, 1
+    cur  = v[1]
+    @inbounds while h < n
+        h     += 1                  # head position
+        cand   = v[h]               # candidate value
+        cand   < cur || continue    # is it new? otherwise skip
+        p     += 1                  # if new store it
+        idx[p] = h
+        cur    = cand               # and update the last seen value
+    end
+    p < n && deleteat!(idx, p+1:n)
+    return idx
+end
+
+"""
+    fprs, tprs, ts = roc_curve(ŷ, y) = roc(ŷ, y)
+
+Return the ROC curve for a two-class probabilistic prediction `ŷ` given the
+ground  truth `y`. The true positive rates, false positive rates over a range
+of thresholds `ts` are returned. Note that if there are `k` unique scores,
+there are correspondingly  `k` thresholds and `k+1` "bins" over which the FPR
+and TPR are constant:
+
+* `[0.0 - thresh[1]]`
+* `[thresh[1] - thresh[2]]`
+* ...
+* `[thresh[k] - 1]`
+
+consequently, `tprs` and `fprs` are of length `k+1` if `ts` is of length `k`.
+
+To draw the curve using your favorite plotting backend, do `plot(fprs, tprs)`.
+"""
+function roc_curve(ŷm, ym)
+    ŷ, y    = skipinvalid(ŷm, ym)
+    n       = length(y)
+    lab_pos = levels(y)[2]
+    scores  = pdf.(ŷ, lab_pos)
+    ranking = sortperm(scores, rev=true)
+
+    scores_sort = scores[ranking]
+    y_sort_bin  = (y[ranking] .== lab_pos)
+
+    idx_unique = _idx_unique_sorted(scores_sort)
+    thresholds = scores_sort[idx_unique]
+
+    # detailed computations with example:
+    # y = [  1   0   0   1   0   0   1]
+    # s = [0.5 0.5 0.2 0.2 0.1 0.1 0.1] thresh are 0.5 0.2, 0.1 // idx [1, 3, 5]
+    # ŷ = [  0   0   0   0   0   0   0] (0.5 - 1.0] # no pos pred
+    # ŷ = [  1   1   0   0   0   0   0] (0.2 - 0.5] # 2 pos pred
+    # ŷ = [  1   1   1   1   0   0   0] (0.1 - 0.2] # 4 pos pred
+    # ŷ = [  1   1   1   1   1   1   1] [0.0 - 0.1] # all pos pre
+
+    idx_unique_2 = idx_unique[2:end]   # [3, 5]
+    n_ŷ_pos      = idx_unique_2 .- 1   # [2, 4] implicit [0, 2, 4, 7]
+
+    cs   = cumsum(y_sort_bin)          # [1, 1, 1, 2, 2, 2, 3]
+    n_tp = cs[n_ŷ_pos]                 # [1, 2] implicit [0, 1, 2, 3]
+    n_fp = n_ŷ_pos .- n_tp             # [1, 2] implicit [0, 1, 2, 4]
+
+    # add end points
+    P = sum(y_sort_bin) # total number of true positives
+    N = n - P           # total number of true negatives
+
+    n_tp = [0, n_tp..., P] # [0, 1, 2, 3]
+    n_fp = [0, n_fp..., N] # [0, 1, 2, 4]
+
+    tprs = n_tp ./ P  # [0/3, 1/3, 2/3, 1]
+    fprs = n_fp ./ N  # [0/4, 1/4, 2/4, 1]
+
+    return fprs, tprs, thresholds
+end
+
+const roc = roc_curve
diff --git a/src/operations.jl b/src/operations.jl
index 584973f8..46e087ce 100644
--- a/src/operations.jl
+++ b/src/operations.jl
@@ -114,7 +114,10 @@ end
 
 ## SURROGATE AND COMPOSITE MODELS
 
-for operation in [:predict, :predict_joint, :transform, :inverse_transform]
+for operation in [:predict,
+                  :predict_joint,
+                  :transform,
+                  :inverse_transform]
     ex = quote
         $operation(model::Union{Composite,Surrogate}, fitresult,X) =
             fitresult.$operation(X)
diff --git a/src/resampling.jl b/src/resampling.jl
index 23598975..e2222683 100644
--- a/src/resampling.jl
+++ b/src/resampling.jl
@@ -31,6 +31,17 @@ err_ambiguous_operation(model, measure) = ArgumentError(
 err_incompatible_prediction_types(model, measure) = ArgumentError(
     _ambiguous_operation(model, measure))
 
+# ==================================================================
+## MODEL TYPES THAT CAN BE EVALUATED
+
+# not exported:
+const Measurable = Union{Supervised,
+                         ProbabilisticSupervisedDetector,
+                         ProbabilisticUnsupervisedDetector,
+                         DeterministicSupervisedDetector,
+                         DeterministicUnsupervisedDetector}
+
+
 # ==================================================================
 ## RESAMPLING STRATEGIES
 
@@ -457,7 +468,7 @@ function Base.show(io::IO, ::MIME"text/plain", e::PerformanceEvaluation)
     println(io, "  measure, measurement, operation, per_fold,\n"*
             "  per_observation, fitted_params_per_fold,\n"*
             "  report_per_fold, train_test_pairs")
-    println(io, "extract:")
+    println(io, "Extract:")
     PrettyTables.pretty_table(io, data, header;
                               header_crayon=PrettyTables.Crayon(bold=false),
                               alignment=:l)
@@ -829,7 +840,7 @@ these properties:
    machine `mach` training in resampling
 
 """
-function evaluate!(mach::Machine{<:Supervised};
+function evaluate!(mach::Machine{<:Measurable};
                    resampling=CV(),
                    measures=nothing,
                    measure=measures,
@@ -842,7 +853,7 @@ function evaluate!(mach::Machine{<:Supervised};
                    repeats=1,
                    force=false,
                    check_measure=true,
-                   verbosity=1)
+                   verbosity=1) where M
 
     # this method just checks validity of options, preprocess the
     # weights, measures, operations, and dispatches a
@@ -896,7 +907,7 @@ wk_options...)`.  See the machine version `evaluate!` for the complete
 list of options.
 
 """
-evaluate(model::Supervised, args...; cache=true, kwargs...) =
+evaluate(model::Measurable, args...; cache=true, kwargs...) =
     evaluate!(machine(model, args...; cache=cache); kwargs...)
 
 # -------------------------------------------------------------------
@@ -1233,7 +1244,7 @@ are not to be confused with any weights bound to a `Resampler` instance
 in a machine, used for training the wrapped `model` when supported.
 
 """
-mutable struct Resampler{S} <: Supervised
+mutable struct Resampler{S} <: Model
     model
     resampling::S # resampling strategy
     measure
diff --git a/src/univariate_finite/arrays.jl b/src/univariate_finite/arrays.jl
index 20e0ee75..46d2381b 100644
--- a/src/univariate_finite/arrays.jl
+++ b/src/univariate_finite/arrays.jl
@@ -249,3 +249,17 @@ function Base.Broadcast.broadcasted(::typeof(mode),
     end
     return reshape(mode_flat, size(u))
 end
+
+## EXTENSION OF CLASSES TO ARRAYS OF UNIVARIATE FINITE
+
+# We already have `classes(::UnivariateFininiteArray)
+
+const ERR_EMPTY_UNIVARIATE_FINITE = ArgumentError(
+    "No `UnivariateFinite` object found from which to extract classes. ")
+
+function MMI.classes(::FullInterface,
+                     yhat::AbstractArray{<:Union{Missing,UnivariateFinite}})
+    i = findfirst(x->!ismissing(x), yhat)
+    i === nothing && throw(ERR_EMPTY_UNIVARIATE_FINITE)
+    return classes(yhat[i])
+end
diff --git a/src/utilities.jl b/src/utilities.jl
index 623dcb4b..fcf9f104 100644
--- a/src/utilities.jl
+++ b/src/utilities.jl
@@ -99,11 +99,27 @@ function recursive_setproperty!(obj, ex::Expr, value)
 end
 
 """
-    check_dimension(X, Y)
+    check_dimensions(X, Y)
+
+Internal function to check two arrays have the same shape.
+
+"""
+@inline function check_dimensions(X, Y)
+    size(X)  == size(Y) ||
+        throw(DimensionMismatch(
+            "Encountered two objects with sizes $(size(X)) and "*
+            "$(size(Y)) which needed to match but don't. "))
+    return nothing
+end
 
-Check that two vectors or matrices have matching dimensions
 """
-function check_dimensions(X::AbstractVecOrMat, Y::AbstractVecOrMat)
+    check_same_nrows(X, Y)
+
+Internal function to check two objects, each a vector or a matrix,
+have the same number of rows.
+
+"""
+@inline function check_same_nrows(X, Y)
     size(X, 1) == size(Y, 1) ||
         throw(DimensionMismatch("The two objects don't have the same " *
                                 "number of rows."))
@@ -117,22 +133,26 @@ Internal function to return a vector or matrix with permuted rows given
 the permutation `perm`.
 """
 function _permute_rows(obj::AbstractVecOrMat, perm::Vector{Int})
-    check_dimensions(obj, perm)
+    check_same_nrows(obj, perm)
     obj isa AbstractVector && return obj[perm]
     obj[perm, :]
 end
 
 """
-shuffle_rows(X::AbstractVecOrMat, Y::AbstractVecOrMat; rng::AbstractRNG = Random.GLOBAL_RNG)
+    shuffle_rows(X::AbstractVecOrMat,
+                 Y::AbstractVecOrMat;
+                 rng::AbstractRNG=Random.GLOBAL_RNG)
+
+Return row-shuffled vectors or matrices using a random permutation of `X`
+and `Y`. An optional random number generator can be specified using
+the `rng` argument.
 
-Return shuffled vectors or matrices using a random permutation of `X` and `Y`. An optional random 
-number generator can be specified using the `rng` argument.
 """
 function shuffle_rows(
     X::AbstractVecOrMat, Y::AbstractVecOrMat;
     rng::AbstractRNG=Random.GLOBAL_RNG
 )
-    check_dimensions(X, Y)
+    check_same_nrows(X, Y)
     perm_length = size(X, 1)
     perm = randperm(rng, perm_length)
     return _permute_rows(X, perm), _permute_rows(Y, perm)
@@ -142,9 +162,9 @@ end
 init_rng(rng)
 
 Create an `AbstractRNG` from `rng`. If `rng` is a non-negative `Integer`, it returns a
-`MersenneTwister` random number generator seeded with `rng`; If `rng` is 
+`MersenneTwister` random number generator seeded with `rng`; If `rng` is
 an `AbstractRNG` object it returns `rng`, otherwise it throws an error.
-""" 
+"""
 function init_rng(rng)
     if (rng isa Integer && rng > 0)
         return Random.MersenneTwister(rng)


Test patch:
diff --git a/test/data/data.jl b/test/data/data.jl
index 230a2edb..33ad3306 100644
--- a/test/data/data.jl
+++ b/test/data/data.jl
@@ -232,6 +232,22 @@ end
         identity.(skipmissing(X[2:end-1,2:end-1]))
 end
 
+@testset "skipinvalid" begin
+    w = rand(5)
+    @test MLJBase.skipinvalid([1, 2, missing, 3, NaN], [missing, 5, 6, 7, 8]) ==
+        ([2, 3], [5, 7])
+    @test(
+        MLJBase._skipinvalid([1, 2, missing, 3, NaN],
+                            [missing, 5, 6, 7, 8],
+                            w) ==
+        ([2, 3], [5, 7], w[[2,4]]))
+    @test(
+        MLJBase._skipinvalid([1, 2, missing, 3, NaN],
+                            [missing, 5, 6, 7, 8],
+                            nothing) ==
+        ([2, 3], [5, 7], nothing))
+end
+
 end # module
 
 true
diff --git a/test/measures/confusion_matrix.jl b/test/measures/confusion_matrix.jl
index 4d6da169..43ae01f0 100644
--- a/test/measures/confusion_matrix.jl
+++ b/test/measures/confusion_matrix.jl
@@ -3,30 +3,61 @@ using MLJBase
 include(joinpath("..", "..", "test", "_models", "models.jl"))
 using .Models
 
+@testset "_categorical" begin
+    a = [1, 1, 2, 3]
+    b = [3, 3, 4, 5]
+    c = [missing, a...]
+    d = [missing, b...]
+    e = categorical(a)
+    f = categorical(b)
+    g = categorical(c)
+    h = categorical(d)
+    rhs = (Set(1:5), Set(1:5))
+    @test Set.(levels.(MLJBase._categorical(a, b))) == rhs
+    @test Set.(levels.(MLJBase._categorical(a, d))) == rhs
+    @test Set.(levels.(MLJBase._categorical(c, b))) == rhs
+    @test Set.(levels.(MLJBase._categorical(c, d))) == rhs
+    @test Set.(levels.(MLJBase._categorical(a, f))) == rhs
+    @test Set.(levels.(MLJBase._categorical(a, h))) == rhs
+    @test Set.(levels.(MLJBase._categorical(b, a))) == rhs
+    @test Set.(levels.(MLJBase._categorical(d, a))) == rhs
+    @test Set.(levels.(MLJBase._categorical(b, c))) == rhs
+    @test Set.(levels.(MLJBase._categorical(d, c))) == rhs
+    @test Set.(levels.(MLJBase._categorical(f, a))) == rhs
+    @test Set.(levels.(MLJBase._categorical(h, a))) == rhs
+end
+
 @testset "basics" begin
-    y = categorical(['m', 'f', 'n', 'f', 'm', 'n', 'n', 'm', 'f'])
-    ŷ = categorical(['f', 'f', 'm', 'f', 'n', 'm', 'n', 'm', 'f'])
+    yraw = ['m',     'm', 'f', 'n', missing, 'f', 'm', 'n', 'n', 'm', 'f']
+    ŷraw = [missing, 'f', 'f', 'm', 'f',     'f', 'n', 'm', 'n', 'm', 'f']
+    y = categorical(yraw)
+    ŷ = categorical(ŷraw)
     l = levels(y) # f, m, n
     cm = MLJBase._confmat(ŷ, y; warn=false)
-    e(l,i,j) = sum((ŷ .== l[i]) .& (y .== l[j]))
+    ŷ_clean, y_clean = MLJBase.skipinvalid(ŷ, y)
+    ee(l,i,j) = sum((ŷ_clean .== l[i]) .& (y_clean .== l[j]))
     for i in 1:3, j in 1:3
-        @test cm[i,j] == e(l,i,j)
+        @test cm[i,j] == ee(l,i,j)
     end
+
+    cm2 = @test_logs (:warn, r"The classes are") MLJBase._confmat(ŷraw, yraw)
+    @test cm2.mat == cm.mat
+
     perm = [3, 1, 2]
     l2 = l[perm]
-    cm2 = MLJBase._confmat(ŷ, y; perm=perm) # no warning because permutation is given
+    cm2 = @test_logs MLJBase._confmat(ŷ, y; perm=perm)
     m = ConfusionMatrix(perm=perm)
     for i in 1:3, j in 1:3
-        @test cm2[i,j] == e(l2,i,j)
+        @test cm2[i,j] == ee(l2,i,j)
     end
-    @test_logs (:warn, "The classes are un-ordered,\nusing order: ['f', 'm', 'n'].\nTo suppress this warning, consider coercing to OrderedFactor.") MLJBase._confmat(ŷ, y)
-    ŷc = coerce(ŷ, OrderedFactor)
-    yc = coerce(y, OrderedFactor)
+    @test_logs (:warn, r"The classes are un") MLJBase._confmat(ŷ, y)
+    ŷc = coerce(ŷ, Union{Missing,OrderedFactor})
+    yc = coerce(y, Union{Missing,OrderedFactor})
     @test MLJBase._confmat(ŷc, yc).mat == cm.mat
 
     y = categorical(['a','b','a','b'])
     ŷ = categorical(['b','b','a','a'])
-    @test_logs (:warn, "The classes are un-ordered,\nusing: negative='a' and positive='b'.\nTo suppress this warning, consider coercing to OrderedFactor.") MLJBase._confmat(ŷ, y)
+    @test_logs (:warn, r"The classes are un") MLJBase._confmat(ŷ, y)
 
     # more tests for coverage
     y = categorical([1,2,3,1,2,3,1,2,3])
diff --git a/test/measures/continuous.jl b/test/measures/continuous.jl
index 79ede171..5f3a3164 100644
--- a/test/measures/continuous.jl
+++ b/test/measures/continuous.jl
@@ -1,20 +1,24 @@
 rng = StableRNG(666899)
 
-@testset "built-in regressor measures" begin
-    y    = [1, 2, 3, 4]
-    yhat = [4, 3, 2, 1]
-    w = [1, 2, 4, 3]
+@testset "regressor measures" begin
+    y    = [1, 42,  2, 3, missing, 4]
+    yhat = [4, NaN, 3, 2, 42,      1]
+    w =    [1, 42,  2, 4, 42,      3]
+    y    = [1,  2, 3, 4]
+    yhat = [4, 3, 2,      1]
+    w =    [1,  2, 4,      3]
     @test isapprox(mae(yhat, y), 2)
     @test isapprox(mae(yhat, y, w), (1*3 + 2*1 + 4*1 + 3*3)/4)
     @test isapprox(rms(yhat, y), sqrt(5))
     @test isapprox(rms(yhat, y, w), sqrt((1*3^2 + 2*1^2 + 4*1^2 + 3*3^2)/4))
-    @test isapprox(mean(l1(yhat, y)), 2)
-    @test isapprox(mean(l1(yhat, y, w)), mae(yhat, y, w))
-    @test isapprox(mean(l2(yhat, y)), 5)
-    @test isapprox(mean(l2(yhat, y, w)), rms(yhat, y, w)^2)
-    @test isapprox(mean(log_cosh(yhat, y)), 1.3715546675)
-
-    yhat = y .+ 1
+    @test isapprox(mean(skipinvalid(l1(yhat, y))), 2)
+    @test isapprox(mean(skipinvalid(l1(yhat, y, w))), mae(yhat, y, w))
+    @test isapprox(mean(skipinvalid(l2(yhat, y))), 5)
+    @test isapprox(mean(skipinvalid(l2(yhat, y, w))), rms(yhat, y, w)^2)
+    @test isapprox(mean(skipinvalid(log_cosh(yhat, y))), 1.3715546675)
+
+    y    = [1, 42,  2, 3, missing, 4]
+    yhat = [2, NaN, 3, 4, 42,      5]
     @test isapprox(rmsl(yhat, y),
                    sqrt((log(1/2)^2 + log(2/3)^2 + log(3/4)^2 + log(4/5)^2)/4))
     @test isapprox(rmslp1(yhat, y),
@@ -23,28 +27,4 @@ rng = StableRNG(666899)
     @test isapprox(mape(yhat, y), (1/1 + 1/2 + 1/3 + 1/4)/4)
 end
 
-@testset "MLJBase.value" begin
-    yhat = randn(rng,5)
-    X = (weight=randn(rng,5), x1 = randn(rng,5))
-    y = randn(rng,5)
-    w = randn(rng,5)
-
-    @test MLJBase.value(mae, yhat, nothing, y, nothing) ≈ mae(yhat, y)
-    @test MLJBase.value(mae, yhat, nothing, y, w) ≈ mae(yhat, y, w)
-
-    spooky(yhat, y) = abs.(yhat - y) |> mean
-    @test MLJBase.value(spooky, yhat, nothing, y, nothing) ≈ mae(yhat, y)
-
-    cool(yhat, y, w) = abs.(yhat - y) .* w |> mean
-    MLJBase.supports_weights(::Type{typeof(cool)}) = true
-    @test MLJBase.value(cool, yhat, nothing, y, w) ≈ mae(yhat, y, w)
-
-    funky(yhat, X, y) = X.weight .* abs.(yhat - y) |> mean
-    MLJBase.is_feature_dependent(::Type{typeof(funky)}) = true
-    @test MLJBase.value(funky, yhat, X, y, nothing) ≈ mae(yhat, y, X.weight)
-
-    weird(yhat, X, y, w) = w .* X.weight .* abs.(yhat - y) |> mean
-    MLJBase.is_feature_dependent(::Type{typeof(weird)}) = true
-    MLJBase.supports_weights(::Type{typeof(weird)}) = true
-    @test MLJBase.value(weird, yhat, X, y, w) ≈ mae(yhat, y, X.weight .* w)
-end
+true
diff --git a/test/measures/finite.jl b/test/measures/finite.jl
index c8429c62..d3b398cb 100644
--- a/test/measures/finite.jl
+++ b/test/measures/finite.jl
@@ -2,7 +2,7 @@ rng = StableRNG(51803)
 
 const Vec = AbstractVector
 
-@testset "built-in classifier measures" begin
+@testset "misclassification_rate" begin
     y    = categorical(collect("asdfasdfaaassdd"))
     yhat = categorical(collect("asdfaadfaasssdf"))
     w = 1:15
@@ -13,60 +13,14 @@ const Vec = AbstractVector
     @test misclassification_rate(yhatm, ym) ≈ 0.2
     @test misclassification_rate(yhat, y, w) ≈ (6*1 + 11*1 + 15*1) / 15
     @test misclassification_rate(yhatm, ym, wm) ≈ (6*1 + 11*1 + 15*1) / 15
-    y = categorical(collect("abb"))
-    L = [y[1], y[2]]
-    d1 = UnivariateFinite(L, [0.1, 0.9]) # a
-    d2 = UnivariateFinite(L, Float32[0.4, 0.6]) # b
-    d3 = UnivariateFinite(L, [0.2, 0.8]) # b
-    yhat = [d1, d2, d3]
-    ym = vcat(y, [missing,])
-    yhatm = vcat(yhat, [d3, ])
-    @test mean(cross_entropy(yhat, y)) ≈
-        Float32(-(log(0.1) + log(0.6) + log(0.8))/3)
-    @test mean(skipmissing(cross_entropy(yhatm, ym))) ≈
-        Float32(-(log(0.1) + log(0.6) + log(0.8))/3)
-    yhat = UnivariateFinite(L, [0.1 0.9;
-                                0.4 0.6;
-                                0.2 0.8])
-    @test isapprox(mean(cross_entropy(yhat, y)),
-                   -(log(0.1) + log(0.6) + log(0.8))/3, atol=eps(Float32))
-    # sklearn test
-    # >>> from sklearn.metrics import log_loss
-    # >>> log_loss(["spam", "ham", "ham", "spam","ham","ham"],
-    #    [[.1, .9], [.9, .1], [.8, .2], [.35, .65], [0.2, 0.8], [0.3,0.7]])
-    # 0.6130097025803921
-    y2 = categorical(["spam", "ham", "ham", "spam", "ham", "ham"])
-    L2 = classes(y2[1])
-    probs = vcat([.1 .9], [.9 .1], [.8 .2], [.35 .65], [0.2 0.8], [0.3 0.7])
-    yhat2 = UnivariateFinite(L2, probs)
-    y2m = vcat(y2, [missing,])
-    yhat2m = UnivariateFinite(L2, vcat(probs, [0.1 0.9]))
-    @test mean(cross_entropy(yhat2, y2)) ≈ 0.6130097025803921
-    @test mean(skipmissing(cross_entropy(yhat2, y2))) ≈ 0.6130097025803921
-    # BrierScore
-    scores = BrierScore()(yhat, y)
-    @test size(scores) == size(y)
-    @test Float32.(scores) ≈ [-1.62, -0.32, -0.08]
-    scoresm = BrierScore()(yhatm, ym)
-    @test Float32.((scoresm)[1:3]) ≈ [-1.62, -0.32, -0.08]
-    @test ismissing(scoresm[end])
-    # sklearn test
-    # >>> from sklearn.metrics import brier_score_loss
-    # >>> brier_score_loss([1, 0, 0, 1, 0, 0], [.9, .1, .2, .65, 0.8, 0.7])
-    # 0.21875 NOTE: opposite orientation
-    @test -mean(BrierScore()(yhat2, y2)) / 2 ≈ 0.21875
-    probs2 = [[.1, .9], [Float32(0.9), Float32(1) - Float32(0.9)], [.8, .2],
-              [.35, .65], [0.2, 0.8], [0.3, 0.7]]
-    yhat3 = [UnivariateFinite(L2, prob) for prob in probs2]
-    @test -mean(BrierScore()(yhat3, y2) / 2) ≈ 0.21875
-    @test mean(BrierLoss()(yhat3, y2) / 2) ≈ -mean(BrierScore()(yhat3, y2) / 2)
 end
 
 @testset "mcr, acc, bacc, mcc" begin
     y = categorical(['m', 'f', 'n', 'f', 'm', 'n', 'n', 'm', 'f'])
     ŷ = categorical(['f', 'f', 'm', 'f', 'n', 'm', 'n', 'm', 'f'])
     @test accuracy(ŷ, y) == 1-mcr(ŷ,y) ==
-            accuracy(MLJBase._confmat(ŷ, y, warn=false))  == 1-mcr(MLJBase._confmat(ŷ, y, warn=false))
+        accuracy(MLJBase._confmat(ŷ, y, warn=false))  ==
+        1-mcr(MLJBase._confmat(ŷ, y, warn=false))
     w = randn(rng,length(y))
     @test accuracy(ŷ, y, w) == 1-mcr(ŷ,y,w)
 
@@ -90,7 +44,8 @@ end
        1.1, 1.8, 0.1, 1.2, 1.8, 1. , 0.1, 0.5, 0.6, 0.7, 0.6, 1.2, 0.6,
        1.2, 0.5, 0.5, 0.8, 0.2, 0.6, 1. , 0.3, 1. , 0.2, 1.1, 1.1, 1.1,
        0.6, 1.4, 1.2, 0.3, 1.1, 0.2, 0.5, 1.6, 0.3, 1. , 0.3, 0.9, 0.9,
-       0. , 0.6, 0.6, 0.4, 0.5, 0.4, 0.2, 0.9, 0.4]
+         0. , 0.6, 0.6, 0.4, 0.5, 0.4, 0.2, 0.9, 0.4]
+
     sk_bacc_w = 0.1581913163016446
     @test bacc(ŷ, y, w) ≈ sk_bacc_w
 
@@ -105,56 +60,6 @@ end
     @test mcc(cm) ≈ 0.39312321239417797
 end
 
-@testset "AUC" begin
-    # this is random binary and random scores generated with numpy
-    # then using roc_auc_score from sklearn to get the AUC
-    # we check that we recover a comparable AUC and that it's invariant
-    # to ordering.
-    c = ["neg", "pos"]
-    y = categorical(c[[0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0,
-                     1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 1,
-                     1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1,
-                     1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0,
-                     1, 0] .+ 1])
-    probs = [
-        0.90237535, 0.41276349, 0.94511611, 0.08390761, 0.55847392,
-        0.26043136, 0.78565351, 0.20133953, 0.7404382 , 0.15307601,
-        0.59596716, 0.8169512 , 0.88200483, 0.23321489, 0.94050483,
-        0.27593662, 0.60702176, 0.36427036, 0.35481784, 0.06416543,
-        0.45576954, 0.12354048, 0.79830435, 0.15799818, 0.20981099,
-        0.43451663, 0.24020098, 0.11401055, 0.25785748, 0.86490263,
-        0.75715379, 0.06550534, 0.12628999, 0.18878245, 0.1283757 ,
-        0.76542903, 0.8780248 , 0.86891113, 0.24835709, 0.06528076,
-        0.72061354, 0.89451634, 0.95634394, 0.07555979, 0.16345437,
-        0.43498831, 0.37774708, 0.31608861, 0.41369339, 0.95691113]
-
-    ŷ = UnivariateFinite(y[1:2], probs, augment=true)
-    # ŷ = [UnivariateFinite(y[1:2], [1.0 - p, p]) for p in [
-    #     0.90237535, 0.41276349, 0.94511611, 0.08390761, 0.55847392,
-    #     0.26043136, 0.78565351, 0.20133953, 0.7404382 , 0.15307601,
-    #     0.59596716, 0.8169512 , 0.88200483, 0.23321489, 0.94050483,
-    #     0.27593662, 0.60702176, 0.36427036, 0.35481784, 0.06416543,
-    #     0.45576954, 0.12354048, 0.79830435, 0.15799818, 0.20981099,
-    #     0.43451663, 0.24020098, 0.11401055, 0.25785748, 0.86490263,
-    #     0.75715379, 0.06550534, 0.12628999, 0.18878245, 0.1283757 ,
-    #     0.76542903, 0.8780248 , 0.86891113, 0.24835709, 0.06528076,
-    #     0.72061354, 0.89451634, 0.95634394, 0.07555979, 0.16345437,
-    #     0.43498831, 0.37774708, 0.31608861, 0.41369339, 0.95691113]]
-    @test isapprox(auc(ŷ, y), 0.455716, rtol=1e-4)
-    ŷ_unwrapped = [ŷ...]
-    @test isapprox(auc(ŷ_unwrapped, y), 0.455716, rtol=1e-4)
-
-    # reversing the roles of positive and negative should return very
-    # similar score
-    y2 = deepcopy(y);
-    levels!(y2, reverse(levels(y2)));
-    @test y == y2
-    @test levels(y) != levels(y2)
-    ŷ2 = UnivariateFinite(y2[1:2], probs, augment=true) # same probs
-    @test isapprox(auc(ŷ2, y2), auc(ŷ, y), rtol=1e-4)
-
-end
-
 @testset "confusion matrix {2}" begin
     # first class is 1 is assumed negative, second positive
     y = categorical([1, 2, 1, 2, 1, 1, 2])
@@ -192,8 +97,8 @@ end
     @test @test_logs((:warn, r"The classes are un-ordered"),
                      recall(ŷ, y) == TP / (TP + FN))
 
-    ŷ = coerce(ŷ, OrderedFactor)
-    y = coerce(y, OrderedFactor)
+    ŷ = coerce(ŷ, Union{Missing,OrderedFactor})
+    y = coerce(y, Union{Missing,OrderedFactor})
 
     @test precision(ŷ, y)   == TP / (TP + FP)
     @test specificity(ŷ, y) == TN / (TN + FP)
@@ -334,7 +239,9 @@ end
     end
     e = info(auc)
     @test e.name == "AreaUnderCurve"
-    @test e.target_scitype == AbstractArray{<:Union{Missing,Finite{2}}}
+    @test e.target_scitype ==
+        Union{AbstractArray{<:Union{Missing,Multiclass{2}}},
+              AbstractArray{<:Union{Missing,OrderedFactor{2}}}}
     @test e.prediction_type == :probabilistic
     @test e.reports_each_observation == false
     @test e.is_feature_dependent == false
@@ -364,12 +271,13 @@ end
 end
 
 @testset "More binary metrics" begin
-    y = coerce(categorical([1, 2, 1, 2, 1, 1, 2, 1, 2, 2, 2, 1, 2,
-                            2, 1, 2, 1, 1, 1, 2, 1, 2, 2, 1, 2, 1,
-                            2, 2, 2]), OrderedFactor)
-    ŷ = coerce(categorical([1, 2, 2, 2, 2, 1, 2, 2, 1, 2, 2, 1, 2,
-                            1, 1, 1, 2, 2, 1, 2, 1, 2, 2, 2, 1, 2,
-                            1, 2, 2]), OrderedFactor)
+    y = coerce([missing, 1, 2, 1, 2, 1, 1, 2, 1, 2, 2, 2, 1, 2,
+                2, 1, 2, 1, 1, 1, 2, 1, 2, 2, 1, 2, 1,
+                2, 2, 2, 1], Union{Missing,OrderedFactor})
+    ŷ = coerce([1, 1, 2, 2, 2, 2, 1, 2, 2, 1, 2, 2, 1, 2,
+                1, 1, 1, 2, 2, 1, 2, 1, 2, 2, 2, 1, 2,
+                1, 2, 2, missing], Union{Missing,OrderedFactor})
+
     # check all constructors
     m = TruePositive()
     @test m(ŷ, y) == truepositive(ŷ, y)
@@ -411,7 +319,8 @@ end
     m = PPV()
     @test m(ŷ, y) == precision(ŷ, y) == ppv(ŷ, y)
     m = Recall()
-    @test m(ŷ, y) == tpr(ŷ, y) == recall(ŷ, y) == sensitivity(ŷ, y) == hit_rate(ŷ, y)
+    @test m(ŷ, y) == tpr(ŷ, y) == recall(ŷ, y) ==
+        sensitivity(ŷ, y) == hit_rate(ŷ, y)
     m = Specificity()
     @test m(ŷ, y) == tnr(ŷ, y) == specificity(ŷ, y) == selectivity(ŷ, y)
     # 'higher order'
@@ -437,12 +346,12 @@ end
 end
 
 @testset "More multiclass metrics" begin
-    y = coerce(categorical([1, 2, 0, 2, 1, 0, 0, 1, 2, 2, 2, 1, 2,
+    y = coerce(categorical([missing, 1, 2, 0, 2, 1, 0, 0, 1, 2, 2, 2, 1, 2,
                             2, 1, 0, 1, 1, 1, 2, 1, 2, 2, 1, 2, 1,
-                            2, 2, 2]), Multiclass)
-    ŷ = coerce(categorical([2, 0, 2, 2, 2, 0, 1, 2, 1, 2, 0, 1, 2,
+                            2, 2, 2, 0]), Union{Missing,Multiclass})
+    ŷ = coerce(categorical([0, 2, 0, 2, 2, 2, 0, 1, 2, 1, 2, 0, 1, 2,
                             1, 1, 1, 2, 0, 1, 2, 1, 2, 2, 2, 1, 2,
-                            1, 2, 2]), Multiclass)
+                            1, 2, 2, missing]), Union{Missing,Multiclass})
     w = Dict(0=>1, 1=>2, 2=>3) #class_w
     # check all constructors
     m = MulticlassTruePositive()
@@ -607,20 +516,6 @@ end
     @test MulticlassFScore(β=1/3, average=micro_avg)(yhat, y) ≈ f1_micro
 end
 
-@testset "ROC" begin
-    y = [  0   0   0   1   0   1   1   0] |> vec |> categorical
-    s = [0.0 0.1 0.1 0.1 0.2 0.2 0.5 0.5] |> vec
-    ŷ = [UnivariateFinite(classes(y[1]), [1.0-p, p]) for p in s]
-
-    fprs, tprs, ts = roc(ŷ, y)
-
-    sk_fprs = [0. , 0.2, 0.4, 0.8, 1. ]
-    sk_tprs = [0. , 0.33333333, 0.66666667, 1., 1.]
-
-    @test fprs ≈ sk_fprs
-    @test tprs ≈ sk_tprs
-end
-
 @testset "docstrings coverage" begin
     @test startswith(info(BrierScore()).docstring, "`BrierScore`")
 end
diff --git a/test/measures/loss_functions_interface.jl b/test/measures/loss_functions_interface.jl
index ba7b3763..096847fe 100644
--- a/test/measures/loss_functions_interface.jl
+++ b/test/measures/loss_functions_interface.jl
@@ -7,19 +7,20 @@ pm1(y) = Int8(2) .* (Int8.(MLJBase.int(y))) .- Int8(3)
 const MARGIN_LOSSES = MLJBase.MARGIN_LOSSES
 const DISTANCE_LOSSES = MLJBase.DISTANCE_LOSSES
 
+# using `WeightedSum` instead of `WeightedMean`; see
+# https://github.com/JuliaML/LossFunctions.jl/issues/149
+WeightedSum(w) = LossFunctions.AggMode.WeightedMean(w, normalize=false)
+
 @testset "LossFunctions.jl - binary" begin
     y = categorical(["yes", "yes", "no", "yes"])
     yes, no = y[1], y[3]
     dyes = MLJBase.UnivariateFinite([yes, no], [0.6, 0.4])
     dno =  MLJBase.UnivariateFinite([yes, no], [0.3, 0.7])
     yhat = [dno, dno, dyes, dyes]
-    X = nothing
     w = [1, 2, 3, 4]
 
-    @test MLJBase.value(MLJBase.ZeroOneLoss(), yhat, X, y, nothing) ≈
-        [1, 1, 1, 0]
-    @test MLJBase.value(MLJBase.zero_one_loss, yhat, X, y, w) ≈
-        [1, 2, 3, 0] ./10 .* 4
+    @test MLJBase.ZeroOneLoss()(yhat, y) ≈ [1, 1, 1, 0]
+    @test MLJBase.zero_one_loss(yhat,y, w) ≈ [1, 2, 3, 0]
 
     N = 10
     y = categorical(rand(rng, ["yes", "no"], N), ordered=true)
@@ -31,17 +32,15 @@ const DISTANCE_LOSSES = MLJBase.DISTANCE_LOSSES
     yhat = MLJBase.UnivariateFinite([no, yes], p_vec, augment=true)
     yhatm = MLJBase._scale.(p_vec) # predictions for raw LossFunctions measure
     w = rand(rng, N)
-    X = nothing
 
     for M_ex in MARGIN_LOSSES
         m = eval(:(MLJBase.$M_ex()))
-        @test MLJBase.value(m, yhat, X, y, nothing) ≈
-            LossFunctions.value(getfield(m, :loss), ym, yhatm) ≈
-            m(yhat, y)
-        @test mean(MLJBase.value(m, yhat, X, y, w)) ≈
-            LossFunctions.value(getfield(m, :loss), ym, yhatm,
-                                LossFunctions.AggMode.WeightedMean(w)) ≈
-                                    mean(m(yhat, y, w))
+        @test m(yhat, y) ≈ LossFunctions.value(getfield(m, :loss), ym, yhatm)
+        @test MLJBase.Mean()(m(yhat, y, w)) ≈
+            LossFunctions.value(getfield(m, :loss),
+                                ym,
+                                yhatm,
+                                WeightedSum(w))/N
     end
 end
 
@@ -57,12 +56,11 @@ end
         m = eval(:(MLJBase.$M_ex()))
         m_ex = MLJBase.snakecase(M_ex)
         @test m == eval(:(MLJBase.$m_ex))
-        @test MLJBase.value(m, yhat, X, y, nothing) ≈
-            LossFunctions.value(getfield(m, :loss), y, yhat) ≈
-            m(yhat, y)
-        @test mean(MLJBase.value(m, yhat, X, y, w)) ≈
+        @test m(yhat, y) ≈
+            LossFunctions.value(getfield(m, :loss), y, yhat)
+        @test mean(m(yhat ,y, w)) ≈
             LossFunctions.value(getfield(m, :loss), y, yhat,
-                                LossFunctions.AggMode.WeightedMean(w)) ≈
-                                    mean(m(yhat ,y, w))
+                                WeightedSum(w))/N
+
     end
 end
diff --git a/test/measures/measure_search.jl b/test/measures/measure_search.jl
index 1b777292..f8aa5e4d 100644
--- a/test/measures/measure_search.jl
+++ b/test/measures/measure_search.jl
@@ -14,7 +14,8 @@ end
     end
 end
 
-task(m) = AbstractVector{<:Finite} <: m.target_scitype
+S = AbstractVector{Union{Missing,Multiclass{3}}}
+task(m) = S <: m.target_scitype
 
 ms = map(measures(task)) do m
     m.name
@@ -29,7 +30,7 @@ ms = map(measures(task)) do m
     m.name
 end
 
-@test !("LogLoss" in ms)
+@test !("Accuracy" in ms)
 @test "RootMeanSquaredError"  in ms
 
 ms = map(measures("Brier")) do  m
@@ -37,4 +38,5 @@ ms = map(measures("Brier")) do  m
 end
 
 @test Set(ms) == Set(["BrierLoss", "BrierScore"])
+
 true
diff --git a/test/measures/measures.jl b/test/measures/measures.jl
index 43f9850c..94b2802a 100644
--- a/test/measures/measures.jl
+++ b/test/measures/measures.jl
@@ -8,6 +8,8 @@ import LossFunctions
 using StableRNGs
 using OrderedCollections: LittleDict
 
+rng  = StableRNGs.StableRNG(123)
+
 @testset "aggregation" begin
     v = rand(5)
     @test aggregate(v, mae) ≈ mean(v)
@@ -37,7 +39,33 @@ end
     @test reports_each_observation(auc) == false
     @test is_feature_dependent(auc) == false
 
-    @test MLJBase.distribution_type(BrierScore) == MLJBase.UnivariateFinite
+    @test MLJBase.distribution_type(auc) == MLJBase.UnivariateFinite
+end
+
+@testset "MLJBase.value" begin
+    yhat = randn(rng,5)
+    X = (weight=randn(rng,5), x1 = randn(rng,5))
+    y = randn(rng,5)
+    w = randn(rng,5)
+
+    @test MLJBase.value(mae, yhat, nothing, y, nothing) ≈ mae(yhat, y)
+    @test MLJBase.value(mae, yhat, nothing, y, w) ≈ mae(yhat, y, w)
+
+    spooky(yhat, y) = abs.(yhat - y) |> mean
+    @test MLJBase.value(spooky, yhat, nothing, y, nothing) ≈ mae(yhat, y)
+
+    cool(yhat, y, w) = abs.(yhat - y) .* w |> mean
+    MLJBase.supports_weights(::Type{typeof(cool)}) = true
+    @test MLJBase.value(cool, yhat, nothing, y, w) ≈ mae(yhat, y, w)
+
+    funky(yhat, X, y) = X.weight .* abs.(yhat - y) |> mean
+    MLJBase.is_feature_dependent(::Type{typeof(funky)}) = true
+    @test MLJBase.value(funky, yhat, X, y, nothing) ≈ mae(yhat, y, X.weight)
+
+    weird(yhat, X, y, w) = w .* X.weight .* abs.(yhat - y) |> mean
+    MLJBase.is_feature_dependent(::Type{typeof(weird)}) = true
+    MLJBase.supports_weights(::Type{typeof(weird)}) = true
+    @test MLJBase.value(weird, yhat, X, y, w) ≈ mae(yhat, y, X.weight .* w)
 end
 
 mutable struct DRegressor <: Deterministic end
@@ -56,22 +84,35 @@ mutable struct PClassifier <: Probabilistic end
 MLJBase.target_scitype(::Type{<:PClassifier}) =
     AbstractVector{<:Finite}
 
+mutable struct PRegressor <: Probabilistic end
+MLJBase.target_scitype(::Type{<:PRegressor}) =
+    AbstractVector{<:Continuous}
+
+mutable struct PCountRegressor <: Probabilistic end
+MLJBase.target_scitype(::Type{<:PCountRegressor}) =
+    AbstractVector{<:Count}
+
 @testset "default_measure" begin
     @test MLJBase.default_measure(DRegressor()) == rms
     @test MLJBase.default_measure(D2Regressor()) == rms
     @test MLJBase.default_measure(DClassifier()) == misclassification_rate
-    @test MLJBase.default_measure(PClassifier()) == cross_entropy
+    @test MLJBase.default_measure(PClassifier()) == log_loss
 
     @test MLJBase.default_measure(DRegressor) == rms
     @test MLJBase.default_measure(D2Regressor) == rms
     @test MLJBase.default_measure(DClassifier) == misclassification_rate
-    @test MLJBase.default_measure(PClassifier) == cross_entropy
+    @test MLJBase.default_measure(PClassifier) == log_loss
+
+    @test MLJBase.default_measure(PRegressor) == log_loss
+    @test MLJBase.default_measure(PCountRegressor) == log_loss
 end
 
+include("confusion_matrix.jl")
+include("roc.jl")
 include("continuous.jl")
 include("finite.jl")
+include("probabilistic.jl")
 include("loss_functions_interface.jl")
-include("confusion_matrix.jl")
 
 @testset "show method for measures" begin
     io = IOBuffer()
@@ -81,18 +122,11 @@ include("confusion_matrix.jl")
     end
 end
 
-@testset "missing values in aggregation" begin
-    v =[1, 2, missing, 5]
+@testset "missing and NaN values in aggregation" begin
+    v =[1, 2, missing, 5, NaN]
     @test MLJBase.Sum()(v) == 8
     @test MLJBase.RootMeanSquare()(v) ≈ sqrt((1 + 4 + 25)/3)
-end
-
-@testset "_skipmissing" begin
-    w = rand(4)
-    @test MLJBase._skipmissing([1, 2, missing, 3], [missing, 5, 6, 7]) ==
-        ([2, 3], [5, 7])
-    @test MLJBase._skipmissing([1, 2, missing, 3], [missing, 5, 6, 7], w) ==
-        ([2, 3], [5, 7], w[[2,4]])
+    @test_throws MLJBase.ERR_NOTHING_LEFT_TO_AGGREGATE MLJBase.Mean()(Float32[])
 end
 
 end
diff --git a/test/measures/probabilistic.jl b/test/measures/probabilistic.jl
new file mode 100644
index 00000000..f29b63ff
--- /dev/null
+++ b/test/measures/probabilistic.jl
@@ -0,0 +1,165 @@
+rng = StableRNG(51803)
+using LinearAlgebra
+
+const Vec = AbstractVector
+
+@testset "AUC" begin
+    # this is random binary and random scores generated with numpy
+    # then using roc_auc_score from sklearn to get the AUC
+    # we check that we recover a comparable AUC and that it's invariant
+    # to ordering.
+    c = ["neg", "pos"]
+    y = categorical(c[[0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0,
+                     1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 1,
+                     1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1,
+                     1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0,
+                     1, 0] .+ 1])
+    probs = [
+        0.90237535, 0.41276349, 0.94511611, 0.08390761, 0.55847392,
+        0.26043136, 0.78565351, 0.20133953, 0.7404382 , 0.15307601,
+        0.59596716, 0.8169512 , 0.88200483, 0.23321489, 0.94050483,
+        0.27593662, 0.60702176, 0.36427036, 0.35481784, 0.06416543,
+        0.45576954, 0.12354048, 0.79830435, 0.15799818, 0.20981099,
+        0.43451663, 0.24020098, 0.11401055, 0.25785748, 0.86490263,
+        0.75715379, 0.06550534, 0.12628999, 0.18878245, 0.1283757 ,
+        0.76542903, 0.8780248 , 0.86891113, 0.24835709, 0.06528076,
+        0.72061354, 0.89451634, 0.95634394, 0.07555979, 0.16345437,
+        0.43498831, 0.37774708, 0.31608861, 0.41369339, 0.95691113]
+
+    ŷ = UnivariateFinite(y[1:2], probs, augment=true)
+    # ŷ = [UnivariateFinite(y[1:2], [1.0 - p, p]) for p in [
+    #     0.90237535, 0.41276349, 0.94511611, 0.08390761, 0.55847392,
+    #     0.26043136, 0.78565351, 0.20133953, 0.7404382 , 0.15307601,
+    #     0.59596716, 0.8169512 , 0.88200483, 0.23321489, 0.94050483,
+    #     0.27593662, 0.60702176, 0.36427036, 0.35481784, 0.06416543,
+    #     0.45576954, 0.12354048, 0.79830435, 0.15799818, 0.20981099,
+    #     0.43451663, 0.24020098, 0.11401055, 0.25785748, 0.86490263,
+    #     0.75715379, 0.06550534, 0.12628999, 0.18878245, 0.1283757 ,
+    #     0.76542903, 0.8780248 , 0.86891113, 0.24835709, 0.06528076,
+    #     0.72061354, 0.89451634, 0.95634394, 0.07555979, 0.16345437,
+    #     0.43498831, 0.37774708, 0.31608861, 0.41369339, 0.95691113]]
+    @test isapprox(auc(ŷ, y), 0.455716, rtol=1e-4)
+    ŷ_unwrapped = [ŷ...]
+    @test isapprox(auc(ŷ_unwrapped, y), 0.455716, rtol=1e-4)
+
+    # reversing the roles of positive and negative should return very
+    # similar score
+    y2 = deepcopy(y);
+    levels!(y2, reverse(levels(y2)));
+    @test y == y2
+    @test levels(y) != levels(y2)
+    ŷ2 = UnivariateFinite(y2[1:2], probs, augment=true) # same probs
+    @test isapprox(auc(ŷ2, y2), auc(ŷ, y), rtol=1e-4)
+
+end
+
+@testset "Log, Brier, Spherical - finite case" begin
+    y = categorical(collect("abb"))
+    L = [y[1], y[2]]
+    d1 = UnivariateFinite(L, [0.1, 0.9]) # a
+    d2 = UnivariateFinite(L, Float32[0.4, 0.6]) # b
+    d3 = UnivariateFinite(L, [0.2, 0.8]) # b
+    yhat = [d1, d2, d3]
+    ym = vcat(y, [missing,])
+    yhatm = vcat(yhat, [d3, ])
+
+    @test mean(log_loss(yhat, y)) ≈
+        Float32(-(log(0.1) + log(0.6) + log(0.8))/3)
+    @test mean(skipmissing(log_loss(yhatm, ym))) ≈
+        Float32(-(log(0.1) + log(0.6) + log(0.8))/3)
+    yhat = UnivariateFinite(L, [0.1 0.9;
+                                0.4 0.6;
+                                0.2 0.8])
+    @test isapprox(mean(log_loss(yhat, y)),
+                   -(log(0.1) + log(0.6) + log(0.8))/3, atol=eps(Float32))
+
+    @test log_score(yhat, y) ≈ -log_loss(yhat, y)
+
+    # sklearn test
+    # >>> from sklearn.metrics import log_loss
+    # >>> log_loss(["spam", "ham", "ham", "spam","ham","ham"],
+    #    [[.1, .9], [.9, .1], [.8, .2], [.35, .65], [0.2, 0.8], [0.3,0.7]])
+    # 0.6130097025803921
+    y2 = categorical(["spam", "ham", "ham", "spam", "ham", "ham"])
+    L2 = classes(y2[1])
+    probs = vcat([.1 .9], [.9 .1], [.8 .2], [.35 .65], [0.2 0.8], [0.3 0.7])
+    yhat2 = UnivariateFinite(L2, probs)
+    y2m = vcat(y2, [missing,])
+    yhat2m = UnivariateFinite(L2, vcat(probs, [0.1 0.9]))
+    @test mean(log_loss(yhat2, y2)) ≈ 0.6130097025803921
+    @test mean(skipmissing(log_loss(yhat2, y2))) ≈ 0.6130097025803921
+
+    # Brier
+    scores = BrierScore()(yhat, y)
+    @test size(scores) == size(y)
+    @test Float32.(scores) ≈ [-1.62, -0.32, -0.08]
+    scoresm = BrierScore()(yhatm, ym)
+    @test Float32.((scoresm)[1:3]) ≈ [-1.62, -0.32, -0.08]
+    @test ismissing(scoresm[end])
+    # sklearn test
+    # >>> from sklearn.metrics import brier_score_loss
+    # >>> brier_score_loss([1, 0, 0, 1, 0, 0], [.9, .1, .2, .65, 0.8, 0.7])
+    # 0.21875 NOTE: opposite orientation
+    @test -mean(BrierScore()(yhat2, y2)) / 2 ≈ 0.21875
+    probs2 = [[.1, .9], [Float32(0.9), Float32(1) - Float32(0.9)], [.8, .2],
+              [.35, .65], [0.2, 0.8], [0.3, 0.7]]
+    yhat3 = [UnivariateFinite(L2, prob) for prob in probs2]
+    @test -mean(BrierScore()(yhat3, y2) / 2) ≈ 0.21875
+    @test mean(BrierLoss()(yhat3, y2) / 2) ≈ -mean(BrierScore()(yhat3, y2) / 2)
+
+    # Spherical
+    s = SphericalScore() # SphericalScore(2)
+    norms = [norm(probs[i,:]) for i in 1:size(probs, 1)]
+    @test (pdf.(yhat2, y2) ./ norms) ≈  s(yhat2, y2)
+    # non-performant version:
+    yhat4 = [yhat2...]
+    @test (pdf.(yhat2, y2) ./ norms) ≈  s(yhat4, y2)
+end
+
+@testset "LogScore, BrierScore, SphericalScore - infinite case" begin
+    uniform = Distributions.Uniform(2, 5)
+    betaprime = Distributions.BetaPrime()
+    discrete_uniform = Distributions.DiscreteUniform(2, 5)
+    w = [2, 3]
+
+    # brier
+    yhat = [missing, uniform]
+    @test isapprox(brier_score(yhat, [1.0, 1.0]) |> last, -1/3)
+    @test isapprox(brier_score(yhat, [NaN, 4.0]) |> last,  1/3)
+    @test isapprox(brier_score(yhat, [1.0, 1.0], w) |> last, -1)
+    yhat = [missing, uniform]
+    # issue https://github.com/JuliaStats/Distributions.jl/issues/1392
+    @test_broken isapprox(brier_score(yhat, [missing, 4.0], w), [1,])
+    yhat = [discrete_uniform, discrete_uniform]
+    @test isapprox(brier_score(yhat, [NaN, 1.0]), [-1/4, -1/4,])
+    @test isapprox(brier_score(yhat, [4.0, 4.0]), [1/4, 1/4,])
+
+    # spherical
+    yhat = [uniform, uniform]
+    @test isapprox(spherical_score(yhat, [1.0, 1.0]), [0, 0])
+    @test isapprox(spherical_score(yhat, [NaN, 4.0]), [0, 1/sqrt(3),])
+    # issue https://github.com/JuliaStats/Distributions.jl/issues/1392
+    @test_broken isapprox(spherical_score(yhat, [missing, 4.0], w), [sqrt(3),])
+    @test isapprox(spherical_score(yhat, [4.0, 4.0], w), [2/sqrt(3), sqrt(3),])
+    yhat = [discrete_uniform, discrete_uniform]
+    @test isapprox(spherical_score(yhat, [NaN, 1.0]), [0, 0])
+    @test isapprox(spherical_score(yhat, [4.0, 4.0]), [1/2, 1/2])
+
+    # log
+    yhat = [uniform, uniform]
+    @test isapprox(log_score(yhat, [4.0, 4.0]), [-log(3), -log(3),])
+    @test isapprox(log_score(yhat, [4.0, 4.0], w), [-2*log(27)/3, -log(27)])
+    yhat = [discrete_uniform, discrete_uniform]
+    # issue https://github.com/JuliaStats/Distributions.jl/issues/1392
+    @test_broken  isapprox(log_score(yhat, [missing, 4.0]), [-log(4),])
+
+    log_score([missing, uniform], [4.0, 4.0])
+
+    # errors
+    @test_throws(MLJBase.err_l2_norm(brier_score),
+                 brier_score([betaprime, betaprime], [1.0, 1.0]))
+    s = SphericalScore(alpha=1)
+    @test_throws MLJBase.ERR_UNSUPPORTED_ALPHA s(yhat, [1.0, 1.0])
+end
+
+true
diff --git a/test/measures/roc.jl b/test/measures/roc.jl
new file mode 100644
index 00000000..aaaed8b7
--- /dev/null
+++ b/test/measures/roc.jl
@@ -0,0 +1,13 @@
+@testset "ROC" begin
+    y = [  0   0   0   1   0   1   1   0] |> vec |> categorical
+    s = [0.0 0.1 0.1 0.1 0.2 0.2 0.5 0.5] |> vec
+    ŷ = UnivariateFinite([0, 1], s, augment=true, pool=y)
+
+    fprs, tprs, ts = roc(ŷ, y)
+
+    sk_fprs = [0. , 0.2, 0.4, 0.8, 1. ]
+    sk_tprs = [0. , 0.33333333, 0.66666667, 1., 1.]
+
+    @test fprs ≈ sk_fprs
+    @test tprs ≈ sk_tprs
+end
diff --git a/test/resampling.jl b/test/resampling.jl
index a484c4fd..8f085b93 100644
--- a/test/resampling.jl
+++ b/test/resampling.jl
@@ -455,7 +455,8 @@ end
     d = Distributions.fit(MLJBase.UnivariateFinite, y)
     pairs = MLJBase.train_test_pairs(scv, 1:10N, nothing, y)
     folds = vcat(first.(pairs), last.(pairs))
-    @test all([Distributions.fit(MLJBase.UnivariateFinite, y[fold]) ≈ d for fold in folds])
+    @test all([Distributions.fit(MLJBase.UnivariateFinite, y[fold]) ≈
+               d for fold in folds])
 end
 
 @testset_accelerated "sample weights in evaluation" accel begin
@@ -716,14 +717,14 @@ end
 
         #resampler as a machine with class weights specified
         cweval = Dict(zip(levels(y), rand(length(levels(y)))));
-        resampler = Resampler(model=model, resampling=CV();
+        resampler = Resampler(model=model, resampling=CV(nfolds=2);
                               measure=MulticlassFScore(return_type=Vector),
                               class_weights=cweval, acceleration=accel)
         resampling_machine = machine(resampler, X, y, cache=false)
         fit!(resampling_machine, verbosity=verb)
         e1   = evaluate(resampling_machine).measurement[1]
         mach = machine(model, X, y, cache=cache)
-        e2   = evaluate!(mach, resampling=CV();
+        e2   = evaluate!(mach, resampling=CV(nfolds=2);
                          measure=MulticlassFScore(return_type=Vector),
                          class_weights=cweval,
                          acceleration=accel,
@@ -735,13 +736,13 @@ end
 
     @testset "warnings about measures not supporting weights" begin
         model = ConstantClassifier()
-        X = (x = [1, 2, 3, 4],)
-        y = categorical(["a", "b", "b", "b"])
-        class_weights = Dict("a"=>0.4, "b"=>0.6)
+        N = 100
+        X, y = make_moons(N)
+        class_weights = Dict(0=>0.4, 1=>0.6)
         @test_logs((:warn, r"Sample weights"),
                    evaluate(model, X, y,
                             resampling=Holdout(fraction_train=0.5),
-                            measure=auc, weights=ones(4)))
+                            measure=auc, weights=ones(N)))
         @test_logs((:warn, r"Class weights"),
                    evaluate(model, X, y,
                             resampling=Holdout(fraction_train=0.5),
diff --git a/test/univariate_finite/arrays.jl b/test/univariate_finite/arrays.jl
index c963613d..401a5144 100644
--- a/test/univariate_finite/arrays.jl
+++ b/test/univariate_finite/arrays.jl
@@ -250,6 +250,15 @@ end
 
 end
 
+@testset "classes" begin
+    v = categorical(collect("abca"), ordered=true)
+    u1 = UnivariateFinite([v[1], v[2]], rand(5), augment=true)
+    @test classes(u1) == collect("abc")
+    u2 = [missing, u1...]
+    @test classes(u2) == collect("abc")
+    @test_throws MLJBase.ERR_EMPTY_UNIVARIATE_FINITE classes(u2[1:1])
+end
+
 end
 
 true
diff --git a/test/utilities.jl b/test/utilities.jl
index 574ab42d..190cf052 100644
--- a/test/utilities.jl
+++ b/test/utilities.jl
@@ -59,10 +59,10 @@ end
     x = randn(rng, 5)
     y = randn(rng, 5, 5)
     z = randn(rng, 5, 5)
-    @test MLJBase.check_dimensions(x, y) === nothing
-    @test MLJBase.check_dimensions(z, x) === nothing
-    @test MLJBase.check_dimensions(y, z) === nothing
-    @test_throws DimensionMismatch MLJBase.check_dimensions(x, randn(rng, 4))
+    @test MLJBase.check_same_nrows(x, y) === nothing
+    @test MLJBase.check_same_nrows(z, x) === nothing
+    @test MLJBase.check_same_nrows(y, z) === nothing
+    @test_throws DimensionMismatch MLJBase.check_same_nrows(x, randn(rng, 4))
 
     x = 1:5 |> collect
     y = 1:5 |> collect