Repository: sunpy/sunpy

Issue:
get_url_for_time_range function in stereo.py in dataretriever not working correctly.
The following query:-

``` python
from sunpy.time.timerange import TimeRange
from sunpy.net.vso.attrs import Time, Instrument
from sunpy.net.dataretriever.client import QueryResponse
import sunpy.net.dataretriever.sources.stereo as stereo

LCClient = stereo.HETClient()
urls = LCClient._get_url_for_timerange(TimeRange('2008/12/01','2010/12/01'),'ahead', 15*u.min)

```

Should return a non-empty list of urls but instead returns an empty list. Possible problem stems from the implementation of scraper.py in sunpy.util. The scraper doesn't work as intended on http://www.srl.caltech.edu/STEREO/DATA/HET.



Discussion:
Thanks @sudk1896!! I can confirm there's a problem with the scraper.

``` python
from sunpy.time import TimeRange
from sunpy.util import scraper

het_ahead = scraper.Scraper('http://www.srl.caltech.edu/STEREO/DATA/HET/Ahead/1minute/AeH%y%b.1m')
het_ahead.now
# that works!
tr = TimeRange('2007-08-01', '2007-08-10') # just a few days
het_ahead.filelist(tr)
# this doesn't
tr = TimeRange('2007-08-01', '2007-09-10') # more than a month
het_ahead.filelist(tr)
# neither that.
tr = TimeRange('2007-08-01T00:00:00', '2007-09-10T03:00:00') # with time too
het_ahead.filelist(tr)
# nop, same thing!
```

I will try to look into it during the weekend.
Please, add any other archive to this issue that you see failing, so I can see if there's a pattern. 

Thanks!

@dpshelio : Will do. Even I am trying to rectify the issue. Keep you posted.


Resolving patch:
diff --git a/sunpy/util/scraper.py b/sunpy/util/scraper.py
--- a/sunpy/util/scraper.py
+++ b/sunpy/util/scraper.py
@@ -12,12 +12,13 @@
 __all__ = ['Scraper']
 
 # regular expressions to convert datetime format
+# added `%e` as for milliseconds `%f/1000`
 TIME_CONVERSIONS = {'%Y': '\d{4}', '%y': '\d{2}',
-                    '%b': '[A-Z]..', '%B': '\W', '%m': '\d{2}',
+                    '%b': '[A-Z][a-z]{2}', '%B': '\W', '%m': '\d{2}',
                     '%d': '\d{2}', '%j': '\d{3}',
                     '%H': '\d{2}', '%I': '\d{2}',
                     '%M': '\d{2}',
-                    '%S': '\d{2}'}
+                    '%S': '\d{2}', '%e': '\d{3}', '%f': '\d{6}'}
 
 class Scraper(object):
     """
@@ -57,7 +58,13 @@ class Scraper(object):
     """
     def __init__(self, pattern, **kwargs):
         self.pattern = pattern.format(**kwargs)
-        self.now = datetime.datetime.now().strftime(self.pattern)
+        milliseconds = re.search('\%e', self.pattern)
+        if not milliseconds:
+            self.now = datetime.datetime.now().strftime(self.pattern)
+        else:
+            now = datetime.datetime.now()
+            milliseconds_ = int(now.microsecond / 1000.)
+            self.now = now.strftime(self.pattern[0:milliseconds.start()] + str(milliseconds_) + self.pattern[milliseconds.end():])
 
     def matches(self, filepath, date):
         return date.strftime(self.pattern) == filepath
@@ -115,9 +122,8 @@ def _extractDateURL(self, url):
         url_to_list = lambda txt: re.sub(r'\.|_', '/', txt).split('/')
         pattern_list = url_to_list(self.pattern)
         url_list = url_to_list(url)
-
         time_order = ['%Y', '%y', '%b', '%B', '%m', '%d', '%j',
-                      '%H', '%I', '%M', '%S']
+                      '%H', '%I', '%M', '%S', '%e', '%f']
         final_date = []
         final_pattern = []
         # Find in directory and filename
@@ -139,10 +145,13 @@ def _extractDateURL(self, url):
         #   Create new empty lists
         final_date = list()
         final_pattern = list()
+        re_together = re_together.replace('[A-Z]', '\\[A-Z]')
         for p,r in zip(pattern_together.split('%')[1:], re_together.split('\\')[1:]):
-            regexp = '\\{}'.format(r)
+            if p == 'e':
+                continue
+            regexp = '\\{}'.format(r) if not r.startswith('[') else r
             pattern = '%{}'.format(p)
-            date_part = re.match(regexp, date_together)
+            date_part = re.search(regexp, date_together)
             date_together = date_together[:date_part.start()] + \
                             date_together[date_part.end():]
             if pattern not in final_pattern:
@@ -182,7 +191,7 @@ def filelist(self, timerange):
             try:
                 opn = urlopen(directory)
                 try:
-                    soup = BeautifulSoup(opn)
+                    soup = BeautifulSoup(opn, "lxml")
                     for link in soup.find_all("a"):
                         href = link.get("href")
                         if href.endswith(self.pattern.split('.')[-1]):


Test patch:
diff --git a/sunpy/util/tests/test_scraper.py b/sunpy/util/tests/test_scraper.py
--- a/sunpy/util/tests/test_scraper.py
+++ b/sunpy/util/tests/test_scraper.py
@@ -12,6 +12,7 @@
     ('%b%y', datetime.timedelta(days=31)),
     ('%m%y', datetime.timedelta(days=31)),
     ('%H%d', datetime.timedelta(hours=1)),
+    ('%y%b', datetime.timedelta(days=31)),
 ]
 
 def testDirectoryDatePattern():
@@ -46,6 +47,12 @@ def testDirectoryRangeFalse():
     timerange = TimeRange('2009/12/30', '2010/01/03')
     assert s.range(timerange) != directory_list
 
+def testNoDateDirectory():
+    s = Scraper('mySpacecraft/myInstrument/xMinutes/aaa%y%b.ext')
+    directory_list = ['mySpacecraft/myInstrument/xMinutes/']
+    timerange = TimeRange('2009/11/20', '2010/01/03')
+    assert s.range(timerange) == directory_list
+
 @pytest.mark.parametrize('pattern, mintime', PATTERN_EXAMPLES)
 def test_smallerPattern(pattern, mintime):
     assert  mintime == Scraper('')._smallerPattern(pattern)
@@ -98,6 +105,12 @@ def testURL_pattern():
     assert not s._URL_followsPattern('fd_20130410_231211.fts.gz')
     assert not s._URL_followsPattern('fd_20130410_ar_231211.fts.gz')
 
+def testURL_patternMilliseconds():
+    s = Scraper('fd_%Y%m%d_%H%M%S_%e.fts')
+    assert s._URL_followsPattern('fd_20130410_231211_119.fts')
+    assert not s._URL_followsPattern('fd_20130410_231211.fts.gz')
+    assert not s._URL_followsPattern('fd_20130410_ar_231211.fts.gz')
+
 # Local files don't work
 # def testFilesRange_sameDirectory_local():
 #     s = Scraper('/'.join(['file:/',sunpy.data.test.rootdir,
@@ -126,3 +139,12 @@ def testFilesRange_sameDirectory_remote():
     timerange = TimeRange(startdate, enddate)
     assert len(s.filelist(timerange)) == 0
 
+@pytest.mark.online
+def testFilesRange_sameDirectory_months_remote():
+    pattern = ('http://www.srl.caltech.edu/{spacecraft}/DATA/{instrument}/'
+               'Ahead/1minute/AeH%y%b.1m')
+    s = Scraper(pattern, spacecraft='STEREO', instrument = 'HET')
+    startdate = datetime.datetime(2007, 8, 1)
+    enddate = datetime.datetime(2007, 9, 10)
+    timerange = TimeRange(startdate, enddate)
+    assert len(s.filelist(timerange)) == 2