Repository: opencb/opencga
Issue: Index from FileWSServer does not work properly

When we launch this WS the system stores the VCF into the DB but the annotation tool is not launched. We have the data but we do not have the annotations.

Then, when we use the Fetch WS we get a Null Pointer Exception.

``````
java.lang.NullPointerException
    org.opencb.datastore.core.QueryResult.<init>(QueryResult.java:58)
    org.opencb.opencga.storage.mongodb.variant.MongoDBStudyConfigurationManager.getStudyConfiguration(MongoDBStudyConfigurationManager.java:94)
    org.opencb.opencga.storage.mongodb.variant.DBObjectToSamplesConverter.convertToDataModelType(DBObjectToSamplesConverter.java:135)
    org.opencb.opencga.storage.mongodb.variant.DBObjectToVariantSourceEntryConverter.convertToDataModelType(DBObjectToVariantSourceEntryConverter.java:114)
    org.opencb.opencga.storage.mongodb.variant.DBObjectToVariantConverter.convertToDataModelType(DBObjectToVariantConverter.java:133)
    org.opencb.opencga.storage.mongodb.variant.DBObjectToVariantConverter.convertToDataModelType(DBObjectToVariantConverter.java:35)
    org.opencb.datastore.mongodb.MongoDBCollection._find(MongoDBCollection.java:209)
    org.opencb.datastore.mongodb.MongoDBCollection.find(MongoDBCollection.java:144)
    org.opencb.opencga.storage.mongodb.variant.VariantMongoDBAdaptor.getAllVariants(VariantMongoDBAdaptor.java:117)
    org.opencb.opencga.storage.mongodb.variant.VariantMongoDBAdaptor.getAllVariantsByRegionList(VariantMongoDBAdaptor.java:183)
    org.opencb.opencga.server.FileWSServer.fetch(FileWSServer.java:789)
    sun.reflect.GeneratedMethodAccessor67.invoke(Unknown Source)
    sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    java.lang.reflect.Method.invoke(Method.java:483)
    org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory$1.invoke(ResourceMethodInvocationHandlerFactory.java:81)
    org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:151)
    org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:171)
    org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$ResponseOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:152)
    org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:104)
    org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:387)
    org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:331)
    org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:103)
    org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:271)
    org.glassfish.jersey.internal.Errors$1.call(Errors.java:271)
    org.glassfish.jersey.internal.Errors$1.call(Errors.java:267)
    org.glassfish.jersey.internal.Errors.process(Errors.java:315)
    org.glassfish.jersey.internal.Errors.process(Errors.java:297)
    org.glassfish.jersey.internal.Errors.process(Errors.java:267)
    org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:297)
    org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:254)
    org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:1030)
    org.glassfish.jersey.servlet.WebComponent.service(WebComponent.java:373)
    org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:381)
    org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:344)
    org.glassfish.jersey.servlet.ServletContainer.service(ServletContainer.java:221)
    org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)```
``````


Changed files:
opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsCalculator.java
opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManager.java

diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsCalculator.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsCalculator.java
index 554f61d11e..6eb3f396a2 100644
--- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsCalculator.java
+++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsCalculator.java
@@ -78,7 +78,7 @@ public class VariantStatisticsCalculator {
         List<VariantStatsWrapper> variantStatsWrappers = new ArrayList<>(variants.size());
 
         for (Variant variant : variants) {
-            VariantSourceEntry file = variant.getSourceEntry(studyId, fileId);
+            VariantSourceEntry file = variant.getSourceEntry(fileId, studyId);
             if (file == null) {
                 skippedFiles++;
                 continue;
diff --git a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManager.java b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManager.java
index 8a06ee0d48..69c69f657b 100644
--- a/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManager.java
+++ b/opencga-storage/opencga-storage-core/src/main/java/org/opencb/opencga/storage/core/variant/stats/VariantStatisticsManager.java
@@ -32,7 +32,6 @@ import org.opencb.opencga.storage.core.StudyConfiguration;
 import org.opencb.opencga.storage.core.runner.StringDataWriter;
 import org.opencb.opencga.storage.core.variant.VariantStorageManager;
 import org.opencb.opencga.storage.core.variant.adaptors.VariantDBAdaptor;
-import org.opencb.opencga.storage.core.variant.adaptors.VariantDBIterator;
 import org.opencb.opencga.storage.core.variant.io.VariantDBReader;
 import org.opencb.opencga.storage.core.variant.io.json.VariantStatsJsonMixin;
 import org.slf4j.Logger;
@@ -51,7 +50,6 @@ import java.util.zip.GZIPOutputStream;
  */
 public class VariantStatisticsManager {
 
-    public static final String BATCH_SIZE = "batchSize";
     private String VARIANT_STATS_SUFFIX = ".variants.stats.json.gz";
     private String SOURCE_STATS_SUFFIX = ".source.stats.json.gz";
     private final JsonFactory jsonFactory;
@@ -90,7 +88,7 @@ public class VariantStatisticsManager {
         ObjectWriter sourceWriter = jsonObjectMapper.writerFor(VariantSourceStats.class);
 
         /** Variables for statistics **/
-        int batchSize = options.getInt(BATCH_SIZE, 1000); // future optimization, threads, etc
+        int batchSize = options.getInt(VariantStorageManager.BATCH_SIZE, 1000); // future optimization, threads, etc
         boolean overwrite = options.getBoolean(VariantStorageManager.OVERWRITE_STATS, false);
         List<Variant> variantBatch = new ArrayList<>(batchSize);
         int retrievedVariants = 0;
@@ -199,18 +197,29 @@ public class VariantStatisticsManager {
         int numTasks = 6;
         int batchSize = 100;  // future optimization, threads, etc
         boolean overwrite = false;
+        String fileId;
         if(options != null) { //Parse query options
-            batchSize = options.getInt(BATCH_SIZE, batchSize);
+            batchSize = options.getInt(VariantStorageManager.BATCH_SIZE, batchSize);
+            numTasks = options.getInt(VariantStorageManager.LOAD_THREADS, numTasks);
             overwrite = options.getBoolean(VariantStorageManager.OVERWRITE_STATS, overwrite);
+            fileId = options.getString(VariantStorageManager.FILE_ID);
+        } else {
+            logger.error("missing required fileId in QueryOptions");
+            throw new Exception("createStats: need a fileId to calculate stats from.");
         }
 
-//        reader, tasks and writer
+        VariantSourceStats variantSourceStats = new VariantSourceStats(fileId, Integer.toString(studyConfiguration.getStudyId()));
+
+
+       // reader, tasks and writer
         VariantDBReader reader = new VariantDBReader(studyConfiguration, variantDBAdaptor, options);
         List<ParallelTaskRunner.Task<Variant, String>> tasks = new ArrayList<>(numTasks);
         for (int i = 0; i < numTasks; i++) {
-            tasks.add(new VariantStatsWrapperTask(overwrite, samples, studyConfiguration, options.getString(VariantStorageManager.FILE_ID)));
+            tasks.add(new VariantStatsWrapperTask(overwrite, samples, studyConfiguration, fileId, variantSourceStats));
         }
-        StringDataWriter writer = new StringDataWriter(Paths.get(output.getPath() + VARIANT_STATS_SUFFIX));
+        Path variantStatsPath = Paths.get(output.getPath() + VARIANT_STATS_SUFFIX);
+        logger.info("will write stats to {}", variantStatsPath);
+        StringDataWriter writer = new StringDataWriter(variantStatsPath);
         
         // runner 
         ParallelTaskRunner.Config config = new ParallelTaskRunner.Config(numTasks, batchSize, numTasks*2, false);
@@ -220,7 +229,14 @@ public class VariantStatisticsManager {
         long start = System.currentTimeMillis();
         runner.run();
         logger.info("finishing stats creation, time: {}ms", System.currentTimeMillis() - start);
-        
+
+        // source stats
+        Path fileSourcePath = Paths.get(output.getPath() + SOURCE_STATS_SUFFIX);
+        OutputStream outputSourceStream = getOutputStream(fileSourcePath, options);
+        ObjectWriter sourceWriter = jsonObjectMapper.writerFor(VariantSourceStats.class);
+        outputSourceStream.write(sourceWriter.writeValueAsBytes(variantSourceStats));
+        outputSourceStream.close();
+
         return output;
     }
 
@@ -232,32 +248,47 @@ public class VariantStatisticsManager {
         private String fileId;
         private ObjectMapper jsonObjectMapper;
         private ObjectWriter variantsWriter;
+        private VariantSourceStats variantSourceStats;
 
-        public VariantStatsWrapperTask(boolean overwrite, Map<String, Set<String>> samples, StudyConfiguration studyConfiguration, String fileId) {
+        public VariantStatsWrapperTask(boolean overwrite, Map<String, Set<String>> samples,
+                                       StudyConfiguration studyConfiguration, String fileId,
+                                       VariantSourceStats variantSourceStats) {
             this.overwrite = overwrite;
             this.samples = samples;
             this.studyConfiguration = studyConfiguration;
             this.fileId = fileId;
             jsonObjectMapper = new ObjectMapper(new JsonFactory());
             variantsWriter = jsonObjectMapper.writerFor(VariantStatsWrapper.class);
+            this.variantSourceStats = variantSourceStats;
         }
 
         @Override
         public List<String> apply(List<Variant> variants) {
 
             List<String> strings = new ArrayList<>(variants.size());
-            
+            boolean defaultCohortAbsent = false;
+
             VariantStatisticsCalculator variantStatisticsCalculator = new VariantStatisticsCalculator(overwrite);
             List<VariantStatsWrapper> variantStatsWrappers = variantStatisticsCalculator.calculateBatch(variants, studyConfiguration.getStudyId()+"", fileId, samples);
-            
+
             long start = System.currentTimeMillis();
             for (VariantStatsWrapper variantStatsWrapper : variantStatsWrappers) {
                 try {
                     strings.add(variantsWriter.writeValueAsString(variantStatsWrapper));
+                    if (variantStatsWrapper.getCohortStats().get(VariantSourceEntry.DEFAULT_COHORT) == null) {
+                        defaultCohortAbsent = true;
+                    }
                 } catch (JsonProcessingException e) {
                     e.printStackTrace();
                 }
             }
+            // we don't want to overwrite file stats regarding all samples with stats about a subset of samples. Maybe if we change VariantSource.stats to a map with every subset...
+            if (!defaultCohortAbsent) {
+                synchronized (variantSourceStats) {
+                    variantSourceStats.updateFileStats(variants);
+                    variantSourceStats.updateSampleStats(variants, null);  // TODO test
+                }
+            }
             logger.debug("another batch  of {} elements calculated. time: {}ms", strings.size(), System.currentTimeMillis() - start);
             if (variants.size() != 0) {
                 logger.info("stats created up to position {}:{}", variants.get(variants.size()-1).getChromosome(), variants.get(variants.size()-1).getStart());
@@ -292,7 +323,7 @@ public class VariantStatisticsManager {
         /** Initialize Json parse **/
         JsonParser parser = jsonFactory.createParser(variantInputStream);
 
-        int batchSize = options.getInt(BATCH_SIZE, 1000);
+        int batchSize = options.getInt(VariantStorageManager.BATCH_SIZE, 1000);
         ArrayList<VariantStatsWrapper> statsBatch = new ArrayList<>(batchSize);
         int writes = 0;
         int variantsNumber = 0;