Repository: Giskard-AI/giskard-oss
Issue: Giskard model scan not working 
Root cause: Validation Gap

### Issue Type

Bug

### Source

source

### Giskard Library Version

2.0.0b14

### Giskard Server Version

2.0.0b11

### OS Platform and Distribution

mac os 10.13.6

### Python version

_No response_

### Installed python packages

```shell
absl-py==0.15.0
aiohttp==3.8.1
aiosignal==1.2.0
alabaster==0.7.12
anaconda-client==1.9.0
anaconda-navigator==2.1.4
anaconda-project==0.10.2
anyio==3.5.0
appdirs==1.4.4
applaunchservices==0.2.1
appnope==0.1.2
appscript==1.1.2
argon2-cffi==21.3.0
argon2-cffi-bindings==21.2.0
arrow==1.2.2
asgiref==3.7.2
astroid==2.6.6
astropy==5.0.4
asttokens==2.0.5
astunparse==1.6.3
async-timeout==4.0.1
atomicwrites==1.4.0
attrs==21.4.0
Automat==20.2.0
autopep8==1.6.0
Babel==2.9.1
backcall==0.2.0
backports.functools-lru-cache==1.6.4
backports.tempfile==1.0
backports.weakref==1.0.post1
bcrypt==3.2.0
beautifulsoup4==4.11.1
bentoml==1.0.22
binaryornot==0.4.4
bitarray==2.4.1
bkcharts==0.2
black==19.10b0
bleach==4.1.0
blinker==1.6.2
bokeh==2.4.2
boto3==1.21.32
botocore==1.24.32
Bottleneck==1.3.4
brotlipy==0.7.0
build==0.10.0
cachetools==4.2.2
capstone==5.0.0.post1
cattrs==23.1.2
certifi==2021.10.8
cffi==1.15.0
chardet==4.0.0
charset-normalizer==2.0.4
circus==0.18.0
clang==5.0
click==8.1.3
click-option-group==0.5.6
cloudpickle==2.0.0
clyent==1.2.2
colorama==0.4.4
colorcet==2.0.6
conda==22.11.1
conda-build==3.21.8
conda-content-trust==0+unknown
conda-pack==0.6.0
conda-package-handling==2.0.2
conda_package_streaming==0.7.0
conda-repo-cli==1.0.4
conda-token==0.3.0
conda-verify==3.4.2
constantly==15.1.0
contextlib2==21.6.0
cookiecutter==1.7.3
cryptography==3.4.8
cssselect==1.1.0
cycler==0.11.0
Cython==0.29.28
cytoolz==0.11.0
daal4py==2021.5.0
dask==2022.2.1
databricks-cli==0.17.7
datashader==0.13.0
datashape==0.5.4
debugpy==1.5.1
decorator==5.1.1
deepmerge==1.1.0
defusedxml==0.7.1
Deprecated==1.2.14
diff-match-patch==20200713
distlib==0.3.6
distributed==2022.2.1
docker==6.1.3
docutils==0.17.1
eli5==0.13.0
entrypoints==0.4
et-xmlfile==1.1.0
exceptiongroup==1.1.1
executing==0.8.3
fastjsonschema==2.15.1
ffmpeg==1.4
filelock==3.12.0
flake8==3.9.2
Flask==2.3.2
flatbuffers==1.12
fonttools==4.25.0
frozenlist==1.2.0
fs==2.4.16
fsspec==2022.2.0
future==0.18.2
gast==0.4.0
gensim==4.1.2
giskard==2.0.0b14
gitdb==4.0.10
GitPython==3.1.32
glob2==0.7
gmpy2==2.1.2
google-api-core==1.25.1
google-auth==1.33.0
google-auth-oauthlib==0.4.6
google-cloud-core==1.7.1
google-cloud-storage==1.31.0
google-crc32c==1.1.2
google-pasta==0.2.0
google-resumable-media==1.3.1
googleapis-common-protos==1.53.0
graphviz==0.20.1
greenlet==1.1.1
grpcio==1.51.1
grpcio-status==1.48.2
gunicorn==20.1.0
h11==0.14.0
h5py==3.1.0
HeapDict==1.0.1
holoviews==1.14.8
hvplot==0.7.3
hyperlink==21.0.0
idna==3.3
imagecodecs==2021.8.26
imageio==2.9.0
imagesize==1.3.0
imbalanced-learn==0.10.1
importlib-metadata==6.0.1
incremental==21.3.0
inflection==0.5.1
iniconfig==1.1.1
intake==0.6.5
intervaltree==3.1.0
ipykernel==6.9.1
ipython==8.2.0
ipython-genutils==0.2.0
ipywidgets==7.6.5
isort==5.9.3
itemadapter==0.3.0
itemloaders==1.0.4
itsdangerous==2.1.2
jdcal==1.4.1
jedi==0.18.1
Jinja2==3.1.2
jinja2-time==0.2.0
jmespath==0.10.0
joblib==1.2.0
json5==0.9.6
jsonschema==4.4.0
jupyter==1.0.0
jupyter-client==6.1.12
jupyter-console==6.4.0
jupyter-core==4.9.2
jupyter-server==1.13.5
jupyterlab==3.3.2
jupyterlab-pygments==0.1.2
jupyterlab-server==2.10.3
jupyterlab-widgets==1.0.0
keras==2.6.0
Keras-Preprocessing==1.1.2
keyring==23.4.0
kiwisolver==1.3.2
langdetect==1.0.9
lazy-object-proxy==1.6.0
libarchive-c==2.9
llvmlite==0.40.1
locket==0.2.1
lockfile==0.12.2
lxml==4.8.0
Markdown==3.3.4
markdown-it-py==3.0.0
MarkupSafe==2.1.3
matplotlib==3.5.1
matplotlib-inline==0.1.2
mccabe==0.6.1
mdurl==0.1.2
mistune==0.8.4
mixpanel==4.10.0
mkl-fft==1.3.1
mkl-random==1.2.2
mkl-service==2.4.0
mlflow-skinny==2.4.2
mock==4.0.3
mpmath==1.2.1
msgpack==1.0.2
multidict==5.2.0
multipledispatch==0.6.0
munkres==1.1.4
mypy-extensions==0.4.3
navigator-updater==0.2.1
nbclassic==0.3.5
nbclient==0.5.13
nbconvert==6.4.4
nbformat==5.3.0
nest-asyncio==1.5.5
networkx==2.7.1
nltk==3.7
nose==1.3.7
notebook==6.4.8
numba==0.57.1
numexpr==2.8.1
numpy==1.22.4
numpydoc==1.2
oauthlib==3.2.2
olefile==0.46
opencv-python==3.4.13.47
openpyxl==3.0.9
opentelemetry-api==1.17.0
opentelemetry-instrumentation==0.38b0
opentelemetry-instrumentation-aiohttp-client==0.38b0
opentelemetry-instrumentation-asgi==0.38b0
opentelemetry-sdk==1.17.0
opentelemetry-semantic-conventions==0.38b0
opentelemetry-util-http==0.38b0
opt-einsum==3.3.0
packaging==23.1
pandas==1.4.2
pandocfilters==1.5.0
panel==0.13.0
param==1.12.0
parsel==1.6.0
parso==0.8.3
partd==1.2.0
pathspec==0.7.0
patsy==0.5.2
pefile==2023.2.7
pep8==1.7.1
pexpect==4.8.0
pickleshare==0.7.5
Pillow==9.0.1
pip==23.1.2
pip-requirements-parser==32.0.1
pip-tools==6.13.0
pipenv==2023.6.2
pkginfo==1.8.2
platformdirs==3.5.1
plotly==5.6.0
pluggy==1.0.0
poyo==0.5.0
prometheus-client==0.13.1
prompt-toolkit==3.0.20
Protego==0.1.16
protobuf==3.19.1
psutil==5.8.0
ptyprocess==0.7.0
pure-eval==0.2.2
py==1.11.0
pyasn1==0.4.8
pyasn1-modules==0.2.8
pycodestyle==2.7.0
pycosat==0.6.3
pycparser==2.21
pycryptodome==3.18.0
pyct==0.4.6
pycurl==7.44.1
pydantic==1.10.11
PyDispatcher==2.0.5
pydocstyle==6.1.1
pyerfa==2.0.0
pyflakes==2.3.1
Pygments==2.15.1
PyHamcrest==2.0.2
PyJWT==2.4.0
pylint==2.9.6
pyls-spyder==0.4.0
pyngrok==6.0.0
pynvml==11.5.0
pyodbc==4.0.32
pyOpenSSL==21.0.0
pyparsing==3.0.4
pyproject_hooks==1.0.0
pyrsistent==0.18.0
PySocks==1.7.1
pytest==7.1.1
python-daemon==2.3.2
python-dateutil==2.8.2
python-json-logger==2.0.7
python-lsp-black==1.0.0
python-lsp-jsonrpc==1.0.0
python-lsp-server==1.2.4
python-multipart==0.0.6
python-slugify==5.0.2
python-snappy==0.6.0
pytz==2021.3
pyviz-comms==2.0.2
PyWavelets==1.3.0
PyYAML==6.0
pyzmq==22.3.0
QDarkStyle==3.0.2
qstylizer==0.1.10
QtAwesome==1.0.3
qtconsole==5.3.0
QtPy==2.0.1
queuelib==1.5.0
regex==2022.3.15
requests==2.27.1
requests-file==1.5.1
requests-oauthlib==1.3.1
requests-toolbelt==1.0.0
rich==13.4.2
rope==0.22.0
rsa==4.7.2
Rtree==0.9.7
ruamel.yaml==0.17.21
ruamel.yaml.clib==0.2.6
ruamel-yaml-conda==0.15.100
s3transfer==0.5.0
schema==0.7.5
scikit-image==0.19.2
scikit-learn==1.0.2
scikit-learn-intelex==2021.20220215.132722
scipy==1.7.3
Scrapy==2.6.1
seaborn==0.11.2
Send2Trash==1.8.0
service-identity==18.1.0
setuptools==67.8.0
shap==0.42.0
simple-di==0.1.5
sip==4.19.13
six==1.15.0
slicer==0.0.7
smart-open==5.1.0
smmap==5.0.0
sniffio==1.2.0
snowballstemmer==2.2.0
sortedcollections==2.1.0
sortedcontainers==2.4.0
soupsieve==2.3.1
Sphinx==4.4.0
sphinxcontrib-applehelp==1.0.2
sphinxcontrib-devhelp==1.0.2
sphinxcontrib-htmlhelp==2.0.0
sphinxcontrib-jsmath==1.0.1
sphinxcontrib-qthelp==1.0.3
sphinxcontrib-serializinghtml==1.1.5
spyder==5.1.5
spyder-kernels==2.1.3
SQLAlchemy==1.4.32
sqlparse==0.4.4
stack-data==0.2.0
starlette==0.28.0
statsmodels==0.13.2
sympy==1.10.1
tables==3.6.1
tabulate==0.8.9
TBB==0.2
tblib==1.7.0
tenacity==8.0.1
tensorboard==2.6.0
tensorboard-data-server==0.6.1
tensorboard-plugin-wit==1.8.1
tensorflow==2.6.5
tensorflow-estimator==2.6.0
termcolor==1.1.0
terminado==0.13.1
testpath==0.5.0
text-unidecode==1.3
textdistance==4.2.1
threadpoolctl==2.2.0
three-merge==0.1.1
tifffile==2021.7.2
tinycss==0.4
tldextract==3.2.0
toml==0.10.2
tomli==1.2.2
toolz==0.11.2
tornado==6.1
tqdm==4.64.0
traitlets==5.1.1
Twisted==22.2.0
typed-ast==1.4.3
typing_extensions==4.6.3
ujson==5.1.0
Unidecode==1.2.0
urllib3==1.26.9
uvicorn==0.22.0
virtualenv==20.23.0
virtualenv-clone==0.5.7
w3lib==1.21.0
watchdog==3.0.0
watchfiles==0.19.0
wcwidth==0.2.5
webencodings==0.5.1
websocket-client==0.58.0
Werkzeug==2.3.6
wheel==0.37.1
widgetsnbextension==3.5.2
wrapt==1.12.1
wurlitzer==3.0.2
xarray==0.20.1
xgboost==1.7.5
xlrd==2.0.1
XlsxWriter==3.0.3
xlwings==0.24.9
xxhash==3.2.0
yapf==0.31.0
yarl==1.6.3
youtube-dl==2021.12.17
zict==2.0.0
zipp==3.7.0
zope.interface==5.4.0
zstandard==0.18.0
```


### Current Behaviour?

```shell
I am trying to scan my naive bayes model with giskard 
i ran this code ### scanning the model
scan_results = giskard.scan(giskard_model, giskard_dataset)
display(scan_results) ``` and i got error; i have pasted the errors below.
Meanwhile, i ran my prediction function code  ``` prediction_function(raw_data["SMS"].head())``` without errors and i got this output ([[9.99999774e-01, 2.26230589e-07],
       [9.99999987e-01, 1.27251159e-08],
       [9.99968875e-01, 3.11254276e-05],
       [9.99913667e-01, 8.63332418e-05],
       [9.99624674e-01, 3.75326250e-04]]) ```

This is the code i used to wrap my naive bayes model ``` #wrapping the model with giskard  
giskard_model = giskard.Model(model=prediction_function,name="Spam SMS filter",feature_names=["SMS"],model_type="classification",classification_labels=[0,1])```
To reproduce this same error i will attach a link to the notebook below. 
I think the errors is coming from giskard scan module
```


### Standalone code OR list down the steps to reproduce the issue

```shell
Here is link to the code
Note, i did not add the code to load the dataset i only include the code to train,wrap, and scan my model. 
I am having issues with scanning my model 
https://nbviewer.org/github/cyberholics/Cybersecurity-with-Artificial-intelligence-/blob/main/spam%20filter%20with%20naive%20bayes%20.ipynb
```


### Relevant log output

```shell
--------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
File ~/anaconda3/lib/python3.9/site-packages/giskard/core/model_validation.py:123, in validate_model_execution(model, dataset, deterministic)
    122 try:
--> 123     prediction = model.predict(validation_ds)
    124 except Exception as e:

File ~/anaconda3/lib/python3.9/site-packages/giskard/models/base/model.py:282, in BaseModel.predict(self, dataset)
    281 if get_cache_enabled():
--> 282     raw_prediction = self._predict_from_cache(dataset)
    283 else:

File ~/anaconda3/lib/python3.9/site-packages/giskard/models/base/model.py:337, in BaseModel._predict_from_cache(self, dataset)
    336 raw_prediction = self.predict_df(unpredicted_df)
--> 337 self._cache.set_cache(dataset.row_hashes[missing], raw_prediction)
    338 cached_predictions.loc[missing] = raw_prediction.tolist()

File ~/anaconda3/lib/python3.9/site-packages/giskard/models/cache/cache.py:59, in ModelCache.set_cache(self, keys, values)
     58 for i in range(len(keys)):
---> 59     self.prediction_cache[keys.iloc[i]] = values[i]
     61 if self.id:

IndexError: index 1 is out of bounds for axis 0 with size 1

The above exception was the direct cause of the following exception:

ValueError                                Traceback (most recent call last)
Input In [28], in <cell line: 2>()
      1 ### scanning the model
----> 2 scan_results = giskard.scan(giskard_model, giskard_dataset)
      3 display(scan_results)

File ~/anaconda3/lib/python3.9/site-packages/giskard/scanner/__init__.py:64, in scan(model, dataset, params, only, verbose, raise_exceptions, validation_flags)
     44 """
     45 Scan a model with a dataset.
     46 
   (...)
     61 
     62 """
     63 scanner = Scanner(params, only=only)
---> 64 return scanner.analyze(model, dataset,
     65                        verbose=verbose,
     66                        raise_exceptions=raise_exceptions,
     67                        validation_flags=validation_flags)

File ~/anaconda3/lib/python3.9/site-packages/giskard/scanner/scanner.py:62, in Scanner.analyze(self, model, dataset, verbose, raise_exceptions, validation_flags)
     60 if not model.is_text_generation:
     61     time_start = perf_counter()
---> 62     validate_model(model=model, validate_ds=dataset, validation_flags=validation_flags)
     63     model_validation_time = perf_counter() - time_start
     64 else:

File ~/anaconda3/lib/python3.9/site-packages/pydantic/decorator.py:40, in pydantic.decorator.validate_arguments.validate.wrapper_function()

File ~/anaconda3/lib/python3.9/site-packages/pydantic/decorator.py:134, in pydantic.decorator.ValidatedFunction.call()

File ~/anaconda3/lib/python3.9/site-packages/pydantic/decorator.py:206, in pydantic.decorator.ValidatedFunction.execute()

File ~/anaconda3/lib/python3.9/site-packages/giskard/core/model_validation.py:45, in validate_model(model, validate_ds, validation_flags)
     43 except (ValueError, TypeError) as err:
     44     _track_validation_error(err, model, validate_ds)
---> 45     raise err
     47 print("Your model is successfully validated.")

File ~/anaconda3/lib/python3.9/site-packages/giskard/core/model_validation.py:42, in validate_model(model, validate_ds, validation_flags)
     38 @configured_validate_arguments
     39 def validate_model(model: BaseModel, validate_ds: Optional[Dataset] = None,
     40                    validation_flags: Optional[ValidationFlags] = ValidationFlags()):
     41     try:
---> 42         _do_validate_model(model, validation_flags, validate_ds)
     43     except (ValueError, TypeError) as err:
     44         _track_validation_error(err, model, validate_ds)

File ~/anaconda3/lib/python3.9/site-packages/giskard/core/model_validation.py:102, in _do_validate_model(model, validation_flags, validate_ds)
     98         validate_label_with_target(
     99             model.meta.name, model.meta.classification_labels, target_values, validate_ds.target
    100         )
    101     if validation_flags.model_execution:
--> 102         validate_model_execution(model, validate_ds)
    103 else:  # Classification with target = None
    104     if validation_flags.model_execution:

File ~/anaconda3/lib/python3.9/site-packages/pydantic/decorator.py:40, in pydantic.decorator.validate_arguments.validate.wrapper_function()

File ~/anaconda3/lib/python3.9/site-packages/pydantic/decorator.py:134, in pydantic.decorator.ValidatedFunction.call()

File ~/anaconda3/lib/python3.9/site-packages/pydantic/decorator.py:206, in pydantic.decorator.ValidatedFunction.execute()

File ~/anaconda3/lib/python3.9/site-packages/giskard/core/model_validation.py:125, in validate_model_execution(model, dataset, deterministic)
    123     prediction = model.predict(validation_ds)
    124 except Exception as e:
--> 125     raise ValueError(error_message) from e
    127 # testing one entry
    128 validation_size = min(len(dataset), 1)

ValueError: Invalid prediction_function.
Before using Giskard, please make sure that 'prediction_function(df[feature_names].head())' does not return an error message.
```


Hi @cyberholics, I think there is a problem with your definition of `prediction_function`. This function takes a dataframe as input (in your case it is a dataframe with a single column `SMS`). You need to take this into account when applying your vectorizer:

```py

def prediction_function(df):
    preprocessed_df = vectorizer.transform(df["SMS"])   # note that we select the "SMS" feature here
    return nb_model.predict_proba(preprocessed_df)
 ```   


Thanks @mattbit!! it worked i can continue with my project