Repository: vgteam/sequenceTubeMap (js)

Issue:
Make BED region download lazy to alow thousands of premade BED regions
Right now, when we download a BED file from a URL, we immediately try to download some metadata about each region.

We need to make this lazy instead, so that we only try to download anything about a region when it is actually accessed.

This will be required to support BED files with thousands of regions, which the Lancet team needs.

Pull request:
Lazy Bed Regions Download
- Chunk metadata from the internet is no longer downloaded until region selection
- GetChunkedData now confirms tracks from a bedURL as user might've pressed Go before tracks are fetched
- Closes #353 

Summary of the current process for bedURLs:
- Upon receiving a bedURL, only the bed file is fetched and bedinfo such as regions are returned 
- Upon selection of a bed region, the respective tracks.json file for that chunk is fetched and tracks are loaded into the track picker if applicable
- Upon receiving a request to getChunkedData, if a bedURL is present, the tracks.json file is re-fetched(if necessary) and used instead of the given tracks from the request in case the user hit the GO button before tracks were loaded into the track picker
- getChunkedPath is called when a bed file is included in the request, downloading the remaining chunk if the bed is a URL and returning a chunk path pointing to the location of the download

Patch:
diff --git a/exampleData/cactus_mixed_chunks.bed b/exampleData/cactus_mixed_chunks.bed
new file mode 100644
index 0000000..646d139
--- /dev/null
+++ b/exampleData/cactus_mixed_chunks.bed
@@ -0,0 +1,3 @@
+ref	1	10	region one to ten	chunk-ref-1-20
+ref	10	20	region ten to twenty	chunk-ref-1-20
+ref	2000	3000	region two to three thousand	chunk-ref-2000-3000
\ No newline at end of file
diff --git a/src/common.mjs b/src/common.mjs
index b2ed3d3..ae001cc 100644
--- a/src/common.mjs
+++ b/src/common.mjs
@@ -110,3 +110,23 @@ export function defaultTrackColors(trackType){
     throw new Error("Invalid track type: " + trackType); 
   }
 }
+
+// Accepts a string, returns whether or not the input is a valid http URL we can call fetch on
+export function isValidURL(string) {
+  if (!string) {
+    return false;
+  }
+
+  let url;
+  try {
+    url = new URL(string)
+  } catch(error) {
+    return false;
+  }
+
+  return url.protocol === "http:" || url.protocol === "https:";
+}
+
+export function isEmpty(obj) {
+  return Object.keys(obj).length === 0;
+}
diff --git a/src/components/HeaderForm.js b/src/components/HeaderForm.js
index a801c39..3f4d6f0 100644
--- a/src/components/HeaderForm.js
+++ b/src/components/HeaderForm.js
@@ -10,7 +10,7 @@ import ExampleSelectButtons from "./ExampleSelectButtons";
 import RegionInput from "./RegionInput";
 import TrackPicker from "./TrackPicker";
 import BedFileDropdown from "./BedFileDropdown";
-import { parseRegion, stringifyRegion } from "../common.mjs";
+import { parseRegion, stringifyRegion, isEmpty } from "../common.mjs";
 
 
 // See src/Types.ts
@@ -509,11 +509,17 @@ class HeaderForm extends Component {
     const regionString = regionChr.concat(":", regionStart, "-", regionEnd);
     return regionString;
   };
-  handleRegionChange = (value, tracks) => {
+
+  // In addition to a new region value, also takes tracks and chunk associated with the region
+  // Update current track if the new tracks are valid
+  // Otherwise check if the current bed file is a url, and if tracks can be fetched from said url
+  // Tracks remain unchanged if neither condition is met
+  handleRegionChange = async (value) => {
     // After user selects a region name or coordinates,
-    // update path and region
-    let coords = value;
+    // update path, region, and associated tracks(if applicable)
 
+    // Update path and region
+    let coords = value;
     if (
       this.state.regionInfo.hasOwnProperty("desc") &&
       this.state.regionInfo["desc"].includes(value)
@@ -522,11 +528,50 @@ class HeaderForm extends Component {
       coords = this.getRegionCoords(value);
     }
     this.setState({ region: coords });
+    
+    let coordsToMetaData = {}
+
+    // Construct a concatenated string of possible coords
+    // Set relative meta data to each coord
+    if (this.state.regionInfo && !isEmpty(this.state.regionInfo)) {
+      for (const [index, path] of this.state.regionInfo["chr"].entries()) {
+        const pathWithRegion = path + ":" + this.state.regionInfo.start[index] + "-" + this.state.regionInfo.end[index];
+        coordsToMetaData[pathWithRegion] = {
+          "tracks": this.state.regionInfo.tracks[index],
+          "chunk": this.state.regionInfo.chunk[index]
+        }
+      }
+    }
+
+    // Set to null if any properties are undefined
+    const tracks = coordsToMetaData?.[coords]?.tracks ?? null;
+    const chunk = coordsToMetaData?.[coords]?.chunk ?? null;
 
     // Override current tracks with new tracks from chunk dir
     if (tracks) {
       this.setState({ tracks: tracks });
-      console.log("New tracks have been population");
+      console.log("New tracks have been applied");
+    } else if (this.state.bedFile && chunk) {
+      // Try to retrieve tracks from the server
+      const json = await fetchAndParse(`${this.props.apiUrl}/getChunkTracks`, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({ bedFile: this.state.bedFile, chunk: chunk }),
+      });
+
+      // Replace tracks if request returns non-falsey value
+      if (json.tracks) {
+        this.setState((laterState) => {
+          if (laterState.region === coords) {
+            // The user still has the same region selected, so apply the tracks we now have
+            return { tracks: json.tracks };
+          }
+          // Otherwise, don't apply the downloaded tracks, because they are no longer relevant.
+          // TODO: Save the downloaded tracks in case the user selects the region again?
+        });
+      }
     }
   };
 
diff --git a/src/components/RegionInput.js b/src/components/RegionInput.js
index 45df60f..c5b0309 100644
--- a/src/components/RegionInput.js
+++ b/src/components/RegionInput.js
@@ -3,10 +3,12 @@ import PropTypes from "prop-types";
 import TextField from "@mui/material/TextField";
 import Autocomplete from "@mui/material/Autocomplete";
 import FormHelperText from "@mui/material/FormHelperText";
+import "../config-client.js";
+import "../config-global.mjs";
+import { isEmpty } from "../common.mjs";
 
 // RegionInput: The path and region input box component
 // Responsible for selecting the path/chr and segment of data to look at
-const isEmpty = (obj) => Object.keys(obj).length === 0;
 
 export const RegionInput = ({
   region,
@@ -20,9 +22,6 @@ export const RegionInput = ({
   const pathNamesColon = pathNames.map((name) => name + ":");
   const pathsWithRegion = [];
 
-  // Store possible options from bed file and match them with their respective tracks
-  let optionToTrack = {};
-
   if (regionInfo && !isEmpty(regionInfo)) {
     // Stitch path name + region start and end
     for (const [index, path] of regionInfo["chr"].entries()) {
@@ -31,14 +30,9 @@ export const RegionInput = ({
       );
     }
 
-    // populate optionToTrack with paths with region
-    pathsWithRegion.forEach((path, i) => optionToTrack[path] = regionInfo.tracks[i]);
-
     // Add descriptions
     pathsWithRegion.push(...regionInfo["desc"]);
 
-    // populate optionToTrack with paths with description as keys
-    regionInfo.desc.forEach((desc, i) => optionToTrack[desc] = regionInfo.tracks[i]);
   }
 
   // Autocomplete selectable options
@@ -56,13 +50,7 @@ export const RegionInput = ({
         id="regionInput"    
 
         onInputChange={(event, newInputValue) => {
-          // If an option is selected, should have a match in optionToTrack
-          if (event.target.textContent in optionToTrack) {
-            // also pass tracks associated with the option
-            handleRegionChange(newInputValue, optionToTrack[event.target.textContent]);
-          } else {
-            handleRegionChange(newInputValue, null);
-          }
+          handleRegionChange(newInputValue);
         }}
 
         options={displayRegions}
diff --git a/src/server.mjs b/src/server.mjs
index a080a30..81f1085 100644
--- a/src/server.mjs
+++ b/src/server.mjs
@@ -21,7 +21,7 @@ import { server as WebSocketServer } from "websocket";
 import dotenv from "dotenv";
 import dirname from "es-dirname";
 import { readFileSync, writeFile } from 'fs';
-import { parseRegion, convertRegionToRangeRegion, stringifyRangeRegion, stringifyRegion } from "./common.mjs";
+import { parseRegion, convertRegionToRangeRegion, stringifyRangeRegion, stringifyRegion, isValidURL } from "./common.mjs";
 import { Readable } from "stream";
 import { finished } from "stream/promises";
 import sanitize from "sanitize-filename";
@@ -265,6 +265,52 @@ async function getChunkedData(req, res, next) {
   console.log(`region = ${req.body.region}`);
   console.log(`tracks = ${JSON.stringify(req.body.tracks)}`);
 
+  // This will have a conitg, start, end, or a contig, start, distance
+  let parsedRegion;
+  try {
+    parsedRegion = parseRegion(req.body.region);
+  } catch (e) {
+    // Whatever went wrong in the parsing, it makes the request bad.
+    throw new BadRequestError("Wrong query: " + e.message + " See ? button above.");
+  }
+
+  // There's a chance this request was sent before the proper tracks were fetched
+  // This can happen when the bed file is a url and track names need to be downloaded
+  // Check if there are tracks specified by the bedFile
+  if (req.body.bedFile && req.body.bedFile !== "none") {
+    const chunk = await getChunkName(req.body.bedFile, parsedRegion);
+    const fetchedTracks = await getChunkTracks(req.body.bedFile, chunk);
+
+    // We're always replacing the given tracks if we were able to find tracks from the bed file
+    if (fetchedTracks) {
+      // Color Settings are retained from the initial request
+      // if newly fetched tracks have matching file names
+      // Store current colors and file names
+      const fileToColor = new Map();
+      for (const key of Object.keys(req.body.tracks)) {
+        const track = req.body.tracks[key];
+        fileToColor.set(track["trackFile"], track["trackColorSettings"]);
+      }
+
+      // Replace new track colors if there's a matching file name
+      for (const track of fetchedTracks) {
+        if (fileToColor.has(track["trackFile"])) {
+          track["trackColorSettings"] = fileToColor.get(track["trackFile"]);
+        }
+      }
+
+      // Convert fetchedTracks into an object format the server expects
+      let fetchedTracksObject = fetchedTracks.reduce((accumulator, obj, index) => {
+        accumulator[index] = obj;
+        return accumulator;
+      }, {});
+
+      console.log("Using new fetched tracks", JSON.stringify(fetchedTracksObject));
+      req.body.tracks = fetchedTracksObject;
+    }
+  }
+
+
   // Assign each request a UUID. v1 UUIDs can be very similar for similar
   // timestamps on the same node, but are still guaranteed to be unique within
   // a given nodejs process.
@@ -276,7 +322,6 @@ async function getChunkedData(req, res, next) {
   // This request owns the directory, so clean it up when the request finishes.
   req.rmChunk = true;
 
-
   // We always have an graph file
   const graphFile = getFirstFileOfType(req.body.tracks, fileTypes.GRAPH);
   // We sometimes have a GBWT with haplotypes that override any in the graph file
@@ -309,15 +354,6 @@ async function getChunkedData(req, res, next) {
     console.log("no BED file provided.");
   }
 
-  // This will have a conitg, start, end, or a contig, start, distance
-  let parsedRegion;
-  try {
-    parsedRegion = parseRegion(req.body.region);
-  } catch (e) {
-    // Whatever went wrong in the parsing, it makes the request bad.
-    throw new BadRequestError("Wrong query: " + e.message + " See ? button above.");
-  }
-
   // check the bed file if this region has been pre-fetched
   let chunkPath = "";
   if (req.withBed) {
@@ -706,15 +742,9 @@ function bedChunkLocalPath(bed, chunk) {
   }
 }
 
-// Gets the chunk path from a region specified in a bedfile, which may be a URL
-// or an allowed local path.
-//
-// Also downloads the chunk data if the bed is an URL and it has not been
-// downloaded yet.
-//
-// The returned path is either an allowed path, or an empty string if we are
-// using a BED without a pre-generated chunk for the given region.
-async function getChunkPath(bed, parsedRegion) {
+// Gets the chunk name from a region specifed in a bedfile
+// Returns an empty string if the region is not found within the bed file
+async function getChunkName(bed, parsedRegion) {
   let chunk = "";
   let regionInfo = await getBedRegions(bed);
 
@@ -734,6 +764,20 @@ async function getChunkPath(bed, parsedRegion) {
       }
     }
   }
+  
+  return chunk;
+}
+
+// Gets the chunk path from a region specified in a bedfile, which may be a URL
+// or an allowed local path.
+//
+// Also downloads the chunk data if the bed is an URL and it has not been
+// downloaded yet.
+//
+// The returned path is either an allowed path, or an empty string if we are
+// using a BED without a pre-generated chunk for the given region.
+async function getChunkPath(bed, parsedRegion) {
+  const chunk = await getChunkName(bed, parsedRegion);
 
   if (chunk === "") {
     // There is no pre-generated chunk for this region.
@@ -1188,23 +1232,6 @@ function hashString(str) {
   return createHash("sha256").update(str).digest("hex");
 }
 
-// returns whether or not a string is a valid http url
-function isValidURL(string) {
-  if (!string) {
-    return False;
-  }
-
-  let url;
-
-  try {
-    url = new URL(string)
-  } catch(error) {
-    return false;
-  }
-
-  return url.protocol === "http:" || url.protocol === "https:";
-}
-
 // Given a URL and a filename, download the given URL to that filename. Assumes required directories exist.
 const downloadFile = async(fileURL, destination) => {
   if (!isAllowedPath(destination)) {
@@ -1305,11 +1332,12 @@ const fetchAndValidate = async(url, maxBytes, existingLocation = null) => {
 //
 // includeContent only downloads the tracks.json file when set to false. If
 // true, all files listed in chunk_contents.txt will be downloaded.
+// includeContent is false when we select a region, we only need the track names
+// includeContent is true when the go button is pressed and a getChunkedData request is called
 const retrieveChunk = async(bedURL, chunk, includeContent) => {
   // path to the designated chunk in the temp directory
   const chunkDir = bedChunkLocalPath(bedURL, chunk);
 
-  // TODO: check if this chunk has been downloaded before, to prevent duplicate fetches
   if (!fs.existsSync(chunkDir)) {
     fs.mkdirSync(chunkDir, { recursive: true });
   }
@@ -1360,6 +1388,52 @@ const timeoutController = (seconds) => {
   return controller;
 }
 
+// Expects a bed file and a chunk name
+// Attempts to download tracks associated with the chunk name from the bed file if it is a URL
+// Returns tracks found from local directories as a tracks object
+async function getChunkTracks (bedFile, chunk) {
+  // Download tracks.json file if it is a URL
+  if (isValidURL(bedFile)) {
+    await retrieveChunk(bedFile, chunk, false);
+  }
+
+  // Get the path to where the track is downloaded
+  let chunkPath = bedChunkLocalPath(bedFile, chunk);
+  let track_json = path.resolve(chunkPath, "tracks.json");
+  let tracks = null;
+  // Attempt to read tracks.json and covnert it into a tracks object
+  if (fs.existsSync(track_json)) {
+    // Create string of tracks data
+    const string_data = fs.readFileSync(track_json);
+
+    // Convert to object container like the client component prop types expect
+    tracks = JSON.parse(string_data);
+  }
+
+  return tracks;
+}
+
+// Expects a request with a bed file and a chunk name
+// Returns tracks retrieved from getChunkTracks
+api.post("/getChunkTracks", (req, res, next) => {
+  console.log("received request for chunk tracks");
+  if (!req.body.bedFile || !req.body.chunk) {
+    throw new BadRequestError("Invalid request format", req.body.bedFile, req.body.chunk);
+  }
+  let promise = (async () => {
+    // tracks are falsy if fetch is unsuccessful
+
+    // TODO: This operation needs to hold a reader lock on the upload/download directories.
+    // waiting for lock changes to be merged
+    const tracks = await getChunkTracks(req.body.bedFile, req.body.chunk);
+    res.json({ tracks: tracks });
+  })();
+
+  // schedules next to be called if promise is rejected
+  promise.catch(next);
+
+});
+
 api.post("/getBedRegions", (req, res, next) => {
   // Bridge async functions to Express error handling with next(err). Don't
   // return a promise. 
@@ -1446,16 +1520,12 @@ async function getBedRegions(bed) {
       
       // Work out where it should be locally.
       const chunk_path = bedChunkLocalPath(bed, chunk);
-      // download the tracks file if bed is an url
-      if (isURL) {
-        await retrieveChunk(bed, chunk, false);
-      }
 
+      // See if we have downloaded tracks.json in a previous instance
       let track_json = path.resolve(chunk_path, "tracks.json");
 
-      let tracks_array = [];
-
-      // If json file specifying the tracks exists
+      // If json file specifying the tracks exists, pass its information into a tracks object
+      // future selection of this region won't re-fetch tracks.json
       if (fs.existsSync(track_json)) {
         // Create string of tracks data
         const string_data = fs.readFileSync(track_json);


Test patch:
diff --git a/src/components/RegionInput.test.js b/src/components/RegionInput.test.js
index 2c6449b..93dbf5f 100644
--- a/src/components/RegionInput.test.js
+++ b/src/components/RegionInput.test.js
@@ -60,5 +60,5 @@ test("it calls handleRegionChange when region is changed with new region", async
   await userEvent.clear(input);
   await userEvent.type(input, NEW_REGION);
 
-  expect(handleRegionChangeMock).toHaveBeenLastCalledWith(NEW_REGION, null);
+  expect(handleRegionChangeMock).toHaveBeenLastCalledWith(NEW_REGION);
 });