Repository: pimutils/khal

Issue:
calendar discovery glob doesn't handle recursion
When using a glob'd directory path, I'm encountering an exception with getting it to parse nessted vdirs correctly.

This is the layout of the directory that `vdirsyncer` currently syncs all my calendars to:
```
/home/demi/Calendars/fastmail/
├── Cocoaheads 
│   └── Boston                      # vdir
│
├── Games 
│   ├── General 
│   │   └── Events                  # vdir
│   ├── Splatoon2 
│   │    ├── Salmon-Run             # vdir
│   │    └── Festivals              # vdir
│   └── The100                      # vdir
│
├── Holidays
│   ├── Japan                       # vdir
│   ├── UK                          # vdir
│   └── USA                         # vdir
│
├── Medical
│   └── Reminders                   # vdir
│
├── Personal
│   ├── Deliveries                  # vdir
│   └── Schedule                    # vdir
│
└── Work
    └── Events                      # vdir
```

The problem occurs when I try to specify a search path for one of these top-level directories (`Cocoaheads`/`Games`/`Holidays`/`Medical`/`Personal`/`Work`) where the level of nesting that the vdirs have is not all the same. so in this case the top-level directory `Games` has vdirs at depths of 1 and 2, so when I specify a glob to accommodate that:
```
[[Games]]
path = ~/Calendars/fastmail/Games/*/*
type = discover
readonly = True
```
then `khal` tries to treat everything at a depth of 2 to be a vdir, regardless if it is a file or directory. thus at startup I get an exception thrown because the `ics` files in the highest vdir are trying to be parsed as a vdir. I guess this isn't technically a bug in that regular because i'm telling `khal` that is where the vdirs are, but I don't know how to write the path so that `khal` will search the given path for vdirs and then classify the contents of the found vdirs under one calendar name in the application. if the special glob pattern `**` could be used to do this, or however to denote that kind of behavior, that would be awesome, but thus far I cannot figure out how to do that.

---
Version: `khal v0.10.1`

Comments:

Patch:
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 7021c0c..8b8d2ef 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -20,6 +20,7 @@ not released
   anymore
 * NEW the `configure` command can now set up vdirsyncer
 * NEW better error message for misuses of `at` and `list`
+* NEW `discover` collection type now supports `**` (arbitrary depths)
 
 0.10.5
 ======
diff --git a/khal/settings/khal.spec b/khal/settings/khal.spec
index 85c93df..be60a07 100644
--- a/khal/settings/khal.spec
+++ b/khal/settings/khal.spec
@@ -10,6 +10,7 @@
 # The path to an existing directory where this calendar is saved as a *vdir*.
 # The directory is searched for events or birthdays (see ``type``). The path
 # also accepts glob expansion via `*` or `?` when type is set to discover.
+# `**` means arbitrary depths of directories.
 # This allows for paths such as `~/accounts/*/calendars/*`, where the
 # calendars directory contains vdir directories. In addition, `~/calendars/*`
 # and `~/calendars/default` are valid paths if there exists a vdir in the
diff --git a/khal/settings/utils.py b/khal/settings/utils.py
index bd1281a..f01c26f 100644
--- a/khal/settings/utils.py
+++ b/khal/settings/utils.py
@@ -24,6 +24,7 @@ import datetime as dt
 import glob
 import logging
 import os
+import pathlib
 from os.path import expanduser, expandvars, join
 from typing import Callable, Iterable, Literal, Optional, Union
 
@@ -189,12 +190,25 @@ def get_unique_name(path: str, names: Iterable[str]) -> str:
     return name
 
 
-def get_all_vdirs(path: str) -> Iterable[str]:
+def get_all_vdirs(expand_path: str) -> Iterable[str]:
     """returns a list of paths, expanded using glob
     """
-    # FIXME currently returns a list of all files in path
-    items = glob.glob(path)
-    return items
+    # FIXME currently returns a list of all directories in path
+    # we add an additional / at the end to make sure we are only getting
+    # directories
+    items = glob.glob(f'{expand_path}/', recursive=True)
+    paths = [pathlib.Path(item) for item in sorted(items, key=len, reverse=True)]
+    leaves = set()
+    parents = set()
+    for path in paths:
+        if path in parents:
+            # we have already seen the current directory as the parent of
+            # another directory, so this directory can't be a vdir
+            continue
+        parents.add(path.parent)
+        leaves.add(path)
+    # sort to make sure that auto generated names are always identical
+    return sorted(os.fspath(path) for path in leaves)
 
 
 def get_vdir_type(_: str) -> str:


Test patch:
diff --git a/tests/settings_test.py b/tests/settings_test.py
index 5ca49cc..1d937bd 100644
--- a/tests/settings_test.py
+++ b/tests/settings_test.py
@@ -172,6 +172,7 @@ def metavdirs(tmpdir):
         '/cal4/dircolor/',
         '/cal4/cfgcolor_again/',
         '/cal4/cfgcolor_once_more/',
+        '/singlecollection/',
     ]
     for one in dirstructure:
         os.makedirs(tmpdir + one)
@@ -198,25 +199,30 @@ def test_broken_color(metavdirs):
 
 
 def test_discover(metavdirs):
-    path = metavdirs
-    vdirs = {vdir[len(path):] for vdir in get_all_vdirs(path + '/*/*')}
-    assert vdirs == {
+    test_vdirs = {
         '/cal1/public', '/cal1/private', '/cal2/public',
         '/cal3/home', '/cal3/public', '/cal3/work',
-        '/cal4/cfgcolor', '/cal4/dircolor', '/cal4/cfgcolor_again', '/cal4/cfgcolor_once_more'
+        '/cal4/cfgcolor', '/cal4/dircolor', '/cal4/cfgcolor_again',
+        '/cal4/cfgcolor_once_more',
+        '/singlecollection',
     }
+    path = metavdirs
+    assert test_vdirs == {vdir[len(path):] for vdir in get_all_vdirs(path + '/**/*/')}
+    assert test_vdirs == {vdir[len(path):] for vdir in get_all_vdirs(path + '/**/')}
+    assert test_vdirs == {vdir[len(path):] for vdir in get_all_vdirs(path + '/**/*')}
 
 
 def test_get_unique_name(metavdirs):
     path = metavdirs
-    vdirs = list(get_all_vdirs(path + '/*/*'))
+    vdirs = list(get_all_vdirs(path + '/**/'))
     names = []
     for vdir in sorted(vdirs):
         names.append(get_unique_name(vdir, names))
-    assert names == [
+    assert sorted(names) == sorted([
         'my private calendar', 'my calendar', 'public', 'home', 'public1',
         'work', 'cfgcolor', 'cfgcolor_again', 'cfgcolor_once_more', 'dircolor',
-    ]
+        'singlecollection',
+    ])
 
 
 def test_config_checks(metavdirs):
@@ -260,14 +266,14 @@ def test_config_checks(metavdirs):
             },
             'public1': {
                 'color': None,
-                'path': '/cal2/public',
+                'path': '/cal3/public',
                 'readonly': False,
                 'type': 'calendar',
                 'priority': 10,
             },
             'public': {
                 'color': None,
-                'path': '/cal3/public',
+                'path': '/cal2/public',
                 'readonly': False,
                 'type': 'calendar',
                 'priority': 10,