Repository: spacetelescope/jwql

Issue:
NIRCam 2739 program page crashes
https://jwql.stsci.edu/NIRCam/archive/2739/obs004/ gives a "list index out of range" error and crashes. Looks like an issue with scraping the proposal information 


Discussion:
The real issue here is that program 2739 does not exist. So I guess this issue really becomes "How do we deal with the situation where a user types in a proposal number and makes a typo?" 

We can have text_scrape() return a bunch of "Not available" strings. But then we would also need to adjust the html template for that case. Also, who knows where else along the line this situation will cause an error.

Resolving patch:
diff --git a/jwql/website/apps/jwql/data_containers.py b/jwql/website/apps/jwql/data_containers.py
--- a/jwql/website/apps/jwql/data_containers.py
+++ b/jwql/website/apps/jwql/data_containers.py
@@ -1320,64 +1320,75 @@ def text_scrape(prop_id):
     # Generate url
     url = 'http://www.stsci.edu/cgi-bin/get-proposal-info?id=' + str(prop_id) + '&submit=Go&observatory=JWST'
     html = BeautifulSoup(requests.get(url).text, 'lxml')
-    lines = html.findAll('p')
-    lines = [str(line) for line in lines]
+    not_available = "not available via this interface" in html.text
 
     program_meta = {}
     program_meta['prop_id'] = prop_id
-    program_meta['phase_two'] = '<a href=https://www.stsci.edu/jwst/phase2-public/{}.pdf target="_blank"> Phase Two</a>'
+    if not not_available:
+        lines = html.findAll('p')
+        lines = [str(line) for line in lines]
 
-    if prop_id[0] == '0':
-        program_meta['phase_two'] = program_meta['phase_two'].format(prop_id[1:])
+        program_meta['phase_two'] = '<a href=https://www.stsci.edu/jwst/phase2-public/{}.pdf target="_blank"> Phase Two</a>'
+
+        if prop_id[0] == '0':
+            program_meta['phase_two'] = program_meta['phase_two'].format(prop_id[1:])
+        else:
+            program_meta['phase_two'] = program_meta['phase_two'].format(prop_id)
+
+        program_meta['phase_two'] = BeautifulSoup(program_meta['phase_two'], 'html.parser')
+
+        links = html.findAll('a')
+        proposal_type = links[0].contents[0]
+
+        program_meta['prop_type'] = proposal_type
+
+        # Scrape for titles/names/contact persons
+        for line in lines:
+            if 'Title' in line:
+                start = line.find('</b>') + 4
+                end = line.find('<', start)
+                title = line[start:end]
+                program_meta['title'] = title
+
+            if 'Principal Investigator:' in line:
+                start = line.find('</b>') + 4
+                end = line.find('<', start)
+                pi = line[start:end]
+                program_meta['pi'] = pi
+
+            if 'Program Coordinator' in line:
+                start = line.find('</b>') + 4
+                mid = line.find('<', start)
+                end = line.find('>', mid) + 1
+                pc = line[mid:end] + line[start:mid] + '</a>'
+                program_meta['pc'] = pc
+
+            if 'Contact Scientist' in line:
+                start = line.find('</b>') + 4
+                mid = line.find('<', start)
+                end = line.find('>', mid) + 1
+                cs = line[mid:end] + line[start:mid] + '</a>'
+                program_meta['cs'] = BeautifulSoup(cs, 'html.parser')
+
+            if 'Program Status' in line:
+                start = line.find('<a')
+                end = line.find('</a>')
+                ps = line[start:end]
+
+                # beautiful soupify text to build absolute link
+                ps = BeautifulSoup(ps, 'html.parser')
+                ps_link = ps('a')[0]
+                ps_link['href'] = 'https://www.stsci.edu' + ps_link['href']
+                ps_link['target'] = '_blank'
+                program_meta['ps'] = ps_link
     else:
-        program_meta['phase_two'] = program_meta['phase_two'].format(prop_id)
-
-    program_meta['phase_two'] = BeautifulSoup(program_meta['phase_two'], 'html.parser')
-
-    links = html.findAll('a')
-    proposal_type = links[0].contents[0]
-
-    program_meta['prop_type'] = proposal_type
-
-    # Scrape for titles/names/contact persons
-    for line in lines:
-        if 'Title' in line:
-            start = line.find('</b>') + 4
-            end = line.find('<', start)
-            title = line[start:end]
-            program_meta['title'] = title
-
-        if 'Principal Investigator:' in line:
-            start = line.find('</b>') + 4
-            end = line.find('<', start)
-            pi = line[start:end]
-            program_meta['pi'] = pi
-
-        if 'Program Coordinator' in line:
-            start = line.find('</b>') + 4
-            mid = line.find('<', start)
-            end = line.find('>', mid) + 1
-            pc = line[mid:end] + line[start:mid] + '</a>'
-            program_meta['pc'] = pc
-
-        if 'Contact Scientist' in line:
-            start = line.find('</b>') + 4
-            mid = line.find('<', start)
-            end = line.find('>', mid) + 1
-            cs = line[mid:end] + line[start:mid] + '</a>'
-            program_meta['cs'] = BeautifulSoup(cs, 'html.parser')
-
-        if 'Program Status' in line:
-            start = line.find('<a')
-            end = line.find('</a>')
-            ps = line[start:end]
-
-            # beautiful soupify text to build absolute link
-            ps = BeautifulSoup(ps, 'html.parser')
-            ps_link = ps('a')[0]
-            ps_link['href'] = 'https://www.stsci.edu' + ps_link['href']
-            ps_link['target'] = '_blank'
-            program_meta['ps'] = ps_link
+        program_meta['phase_two'] = 'N/A'
+        program_meta['prop_type'] = 'N/A'
+        program_meta['title'] = 'Proposal not available or does not exist'
+        program_meta['pi'] = 'N/A'
+        program_meta['pc'] = 'N/A'
+        program_meta['cs'] = 'N/A'
+        program_meta['ps'] = 'N/A'
 
     return program_meta
 


Test patch: