Repository: google/turbinia
Pull request #257 (2018-09-21 11:13:19)

Job manager
Second try at https://github.com/google/turbinia/pull/248


Patch:
diff --git a/terraform/gcp/machines.tf b/terraform/gcp/machines.tf
index 43abb35..5fc6a93 100644
--- a/terraform/gcp/machines.tf
+++ b/terraform/gcp/machines.tf
@@ -9,7 +9,7 @@ resource "google_compute_instance" "turbinia-server" {
   # Use default Ubuntu image as operating system.
   boot_disk {
     initialize_params {
-      image = "${var.ubuntu_1604_image}"
+      image = "${var.ubuntu_image}"
     }
   }
 
@@ -37,7 +37,7 @@ resource "google_compute_instance" "turbinia-worker" {
   # Use default Ubuntu image as operating system.
   boot_disk {
     initialize_params {
-      image = "${var.ubuntu_1604_image}"
+      image = "${var.ubuntu_image}"
     }
   }
 
@@ -50,3 +50,41 @@ resource "google_compute_instance" "turbinia-worker" {
   # Provision the machine with a script.
   metadata_startup_script = "${data.template_file.turbinia-worker.rendered}"
 }
+
+# Enabling GCP services individually because the google_project_services
+# resource type will disable services not in the list.
+resource "google_project_service" "project" {
+  project = "${var.project}"
+  service = "cloudfunctions.googleapis.com"
+  disable_on_destroy = false
+}
+
+resource "google_project_service" "project" {
+  project = "${var.project}"
+  service = "compute.googleapis.com"
+  disable_on_destroy = false
+}
+
+resource "google_project_service" "project" {
+  project = "${var.project}"
+  service = "datastore.googleapis.com"
+  disable_on_destroy = false
+}
+
+resource "google_project_service" "project" {
+  project = "${var.project}"
+  service = "iam.googleapis.com"
+  disable_on_destroy = false
+}
+
+resource "google_project_service" "project" {
+  project = "${var.project}"
+  service = "pubsub.googleapis.com"
+  disable_on_destroy = false
+}
+
+resource "google_project_service" "project" {
+  project = "${var.project}"
+  service = "storage-component.googleapis.com"
+  disable_on_destroy = false
+}
diff --git a/terraform/gcp/variables.tf b/terraform/gcp/variables.tf
index 96e9648..bb008db 100644
--- a/terraform/gcp/variables.tf
+++ b/terraform/gcp/variables.tf
@@ -1,4 +1,4 @@
 variable "project"            {}
 variable "region"             {}
 variable "zone"               {}
-variable "ubuntu_1604_image"  { default = "ubuntu-os-cloud/ubuntu-1604-lts" }
+variable "ubuntu_image"  { default = "ubuntu-os-cloud/ubuntu-1804-lts" }
diff --git a/turbinia/evidence.py b/turbinia/evidence.py
index 772baad..94270e6 100644
--- a/turbinia/evidence.py
+++ b/turbinia/evidence.py
@@ -24,6 +24,8 @@ from turbinia import config
 from turbinia import TurbiniaException
 from turbinia.processors import mount_local
 
+# pylint: disable=keyword-arg-before-vararg
+
 config.LoadConfig()
 if config.TASK_MANAGER == 'PSQ':
   from turbinia.processors import google_cloud
@@ -162,6 +164,7 @@ class RawDisk(Evidence):
   """Evidence object for Disk based evidence.
 
   Attributes:
+    losetup_path: Path to the losetup device for this disk.
     mount_path: The mount path for this disk (if any).
     mount_partition: The mount partition for this disk (if any).
     size:  The size of the disk in bytes.
@@ -170,6 +173,7 @@ class RawDisk(Evidence):
   def __init__(self, mount_path=None, mount_partition=None, size=None, *args,
                **kwargs):
     """Initialization for raw disk evidence object."""
+    self.losetup_path = None
     self.mount_path = mount_path
     self.mount_partition = mount_partition
     self.size = size
diff --git a/turbinia/jobs/__init__.py b/turbinia/jobs/__init__.py
index dc33b9a..93f08b4 100644
--- a/turbinia/jobs/__init__.py
+++ b/turbinia/jobs/__init__.py
@@ -13,60 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Turbinia jobs."""
-
-import uuid
-
-
-def get_jobs():
-  """Gets a list of all job objects.
-
-  Returns:
-    A list of TurbiniaJobs.
-  """
-  # Defer imports to prevent circular dependencies during init.
-  from turbinia.jobs.grep import GrepJob
-  from turbinia.jobs.hadoop import HadoopAnalysisJob
-  from turbinia.jobs.plaso import PlasoJob
-  from turbinia.jobs.psort import PsortJob
-  from turbinia.jobs.sshd import SSHDExtractionJob
-  from turbinia.jobs.sshd import SSHDAnalysisJob
-  from turbinia.jobs.strings import StringsJob
-  from turbinia.jobs.worker_stat import StatJob
-  from turbinia.jobs.tomcat import TomcatExtractionJob
-  from turbinia.jobs.tomcat import TomcatAnalysisJob
-  from turbinia.jobs.analysis.http_access_logs import HTTPAccessLogExtractionJob
-  from turbinia.jobs.analysis.http_access_logs import HTTPAccessLogAnalysisJob
-  from turbinia.jobs.analysis.jenkins import JenkinsAnalysisJob
-  # TODO(aarontp): Dynamically look up job objects and make enabling/disabling
-  # configurable through config and/or recipes.
-  return [
-      GrepJob(), HadoopAnalysisJob(), HTTPAccessLogAnalysisJob(),
-      HTTPAccessLogExtractionJob(), JenkinsAnalysisJob(), PlasoJob(),
-      PsortJob(), SSHDExtractionJob(), SSHDAnalysisJob(), StatJob(),
-      StringsJob(), TomcatAnalysisJob(), TomcatExtractionJob()]
-
-
-class TurbiniaJob(object):
-  """Base class for Turbinia Jobs.
-
-  Attributes:
-    name: Name of Job
-    id: Id of job
-    priority: Job priority from 0-100, lowest == highest priority
-  """
-
-  def __init__(self, name=None):
-    self.name = name
-    self.id = uuid.uuid4().hex
-    self.priority = 100
-
-  def create_tasks(self, evidence_):
-    """Create Turbinia tasks to be run.
-
-    Args:
-      evidence_: A list of evidence objects
-
-    Returns:
-      A List of TurbiniaTask objects.
-    """
-    raise NotImplementedError
+from turbinia.jobs import grep
+from turbinia.jobs import hadoop
+from turbinia.jobs import http_access_logs
+from turbinia.jobs import jenkins
+from turbinia.jobs import plaso
+from turbinia.jobs import psort
+from turbinia.jobs import sshd
+from turbinia.jobs import strings
+from turbinia.jobs import tomcat
+from turbinia.jobs import worker_stat
diff --git a/turbinia/jobs/analysis/__init__.py b/turbinia/jobs/analysis/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/turbinia/jobs/grep.py b/turbinia/jobs/grep.py
index 0e9ff90..9fb2014 100644
--- a/turbinia/jobs/grep.py
+++ b/turbinia/jobs/grep.py
@@ -19,19 +19,19 @@ from __future__ import unicode_literals
 from turbinia.evidence import TextFile
 from turbinia.evidence import FilteredTextFile
 from turbinia.evidence import PlasoCsvFile
-from turbinia.jobs import TurbiniaJob
+from turbinia.jobs import interface
+from turbinia.jobs import manager
 from turbinia.workers.grep import GrepTask
 
 
-class GrepJob(TurbiniaJob):
+class GrepJob(interface.TurbiniaJob):
   """Filter input based on regular expression patterns."""
 
   # The types of evidence that this Job will process
   evidence_input = [TextFile, PlasoCsvFile]
   evidence_output = [FilteredTextFile]
 
-  def __init__(self):
-    super(GrepJob, self).__init__(name='GrepJob')
+  NAME = 'GrepJob'
 
   def create_tasks(self, evidence):
     """Create task.
@@ -44,3 +44,6 @@ class GrepJob(TurbiniaJob):
     """
     tasks = [GrepTask() for _ in evidence]
     return tasks
+
+
+manager.JobsManager.RegisterJob(GrepJob)
diff --git a/turbinia/jobs/hadoop.py b/turbinia/jobs/hadoop.py
index aa07137..8ecb407 100644
--- a/turbinia/jobs/hadoop.py
+++ b/turbinia/jobs/hadoop.py
@@ -20,19 +20,19 @@ from turbinia.evidence import GoogleCloudDisk
 from turbinia.evidence import GoogleCloudDiskRawEmbedded
 from turbinia.evidence import RawDisk
 from turbinia.evidence import ReportText
-from turbinia.jobs import TurbiniaJob
+from turbinia.jobs import interface
+from turbinia.jobs import manager
 from turbinia.workers.hadoop import HadoopAnalysisTask
 
 
-class HadoopAnalysisJob(TurbiniaJob):
+class HadoopAnalysisJob(interface.TurbiniaJob):
   """Analyzes Hadoop AppRoot files."""
 
-  # The types of evidence that this Job will process
   evidence_input = [GoogleCloudDisk, GoogleCloudDiskRawEmbedded, RawDisk]
   evidence_output = [ReportText]
 
-  def __init__(self):
-    super(HadoopAnalysisJob, self).__init__(name='HadoopAnalysisJob')
+  NAME = 'HadoopAnalysisJob'
+
 
   def create_tasks(self, evidence):
     """Create task.
@@ -45,3 +45,5 @@ class HadoopAnalysisJob(TurbiniaJob):
     """
     tasks = [HadoopAnalysisTask() for _ in evidence]
     return tasks
+
+manager.JobsManager.RegisterJob(HadoopAnalysisJob)
\ No newline at end of file
diff --git a/turbinia/jobs/analysis/http_access_logs.py b/turbinia/jobs/http_access_logs.py
similarity index 83%
rename from turbinia/jobs/analysis/http_access_logs.py
rename to turbinia/jobs/http_access_logs.py
index 1cebee8..dca51d7 100644
--- a/turbinia/jobs/analysis/http_access_logs.py
+++ b/turbinia/jobs/http_access_logs.py
@@ -23,8 +23,8 @@ from turbinia.evidence import GoogleCloudDisk
 from turbinia.evidence import GoogleCloudDiskRawEmbedded
 from turbinia.evidence import ExportedFileArtifact
 from turbinia.evidence import ReportText
-
-from turbinia.jobs import TurbiniaJob
+from turbinia.jobs import interface
+from turbinia.jobs import manager
 from turbinia.workers.analysis import wordpress
 
 ACCESS_LOG_ARTIFACTS = [
@@ -33,7 +33,7 @@ ACCESS_LOG_ARTIFACTS = [
     'ApacheAccessLogs'
 ]
 
-class HTTPAccessLogExtractionJob(TurbiniaJob):
+class HTTPAccessLogExtractionJob(interface.TurbiniaJob):
   """HTTP Access log extraction job."""
 
   evidence_input = [
@@ -41,9 +41,8 @@ class HTTPAccessLogExtractionJob(TurbiniaJob):
 
   evidence_output = [ExportedFileArtifact]
 
-  def __init__(self):
-    super(HTTPAccessLogExtractionJob, self).__init__(
-        name='HTTPAccessLogExtractionJob')
+  NAME = 'HTTPAccessLogExtractionJob'
+
 
   def create_tasks(self, evidence):
     """Create task.
@@ -60,15 +59,13 @@ class HTTPAccessLogExtractionJob(TurbiniaJob):
                     in evidence])
     return tasks
 
-class HTTPAccessLogAnalysisJob(TurbiniaJob):
+class HTTPAccessLogAnalysisJob(interface.TurbiniaJob):
   """HTTP Access log analysis job."""
-   # Types of evidence that this Job will process.
+
   evidence_input = [ExportedFileArtifact]
   evidence_output = [ReportText]
 
-  def __init__(self):
-    super(HTTPAccessLogAnalysisJob, self).__init__(
-        name='HTTPAccessLogAnalysisJob')
+  NAME = 'HTTPAccessLogAnalysisJob'
 
   def create_tasks(self, evidence):
     """Create task.
@@ -79,3 +76,6 @@ class HTTPAccessLogAnalysisJob(TurbiniaJob):
     """
     evidence = [e for e in evidence if e.artifact_name in ACCESS_LOG_ARTIFACTS]
     return [wordpress.WordpressAccessLogAnalysisTask() for _ in evidence]
+
+manager.JobsManager.RegisterJobs(
+    [HTTPAccessLogExtractionJob, HTTPAccessLogAnalysisJob])
\ No newline at end of file
diff --git a/turbinia/jobs/interface.py b/turbinia/jobs/interface.py
new file mode 100644
index 0000000..e84b23c
--- /dev/null
+++ b/turbinia/jobs/interface.py
@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Interface for Jobs."""
+import uuid
+
+
+class TurbiniaJob(object):
+  """Base class for Turbinia Jobs.
+
+  Attributes:
+    name: Name of Job
+    id: Id of job
+    priority: Job priority from 0-100, lowest == highest priority
+  """
+
+  NAME = 'name'
+
+  def __init__(self):
+    self.name = self.NAME
+    self.id = uuid.uuid4().hex
+    self.priority = 100
+
+  def create_tasks(self, evidence):
+    """Create Turbinia tasks to be run.
+
+    Args:
+      evidence: A list of evidence objects
+
+    Returns:
+      A List of TurbiniaTask objects.
+    """
+    raise NotImplementedError
diff --git a/turbinia/jobs/analysis/jenkins.py b/turbinia/jobs/jenkins.py
similarity index 86%
rename from turbinia/jobs/analysis/jenkins.py
rename to turbinia/jobs/jenkins.py
index e96ea38..8833773 100644
--- a/turbinia/jobs/analysis/jenkins.py
+++ b/turbinia/jobs/jenkins.py
@@ -21,20 +21,19 @@ from turbinia.evidence import RawDisk
 from turbinia.evidence import GoogleCloudDisk
 from turbinia.evidence import GoogleCloudDiskRawEmbedded
 from turbinia.evidence import ReportText
-from turbinia.jobs import TurbiniaJob
+from turbinia.jobs import interface
 from turbinia.workers.analysis.jenkins import JenkinsAnalysisTask
 
 
-class JenkinsAnalysisJob(TurbiniaJob):
+class JenkinsAnalysisJob(interface.TurbiniaJob):
   """Jenkins analysis job."""
 
-  # Types of evidence that this Job will process.
   evidence_input = [
     Directory, RawDisk, GoogleCloudDisk, GoogleCloudDiskRawEmbedded]
   evidence_output = [ReportText]
 
-  def __init__(self):
-    super(JenkinsAnalysisJob, self).__init__(name='JenkinsAnalysisJob')
+  NAME = 'JenkinsAnalysisJob'
+
 
   def create_tasks(self, evidence):
     """Create task for Jenkins analysis job.
diff --git a/turbinia/jobs/manager.py b/turbinia/jobs/manager.py
new file mode 100644
index 0000000..c6f8530
--- /dev/null
+++ b/turbinia/jobs/manager.py
@@ -0,0 +1,136 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""This file contains a class for managing jobs."""
+
+from __future__ import unicode_literals
+
+
+class JobsManager(object):
+  """The jobs manager."""
+
+  _job_classes = {}
+
+  @classmethod
+  def DeregisterJob(cls, job_class):
+    """Deregisters a job class.
+
+    The job classes are identified based on their lower case name.
+
+    Args:
+      job_class (type): class object of the job.
+
+    Raises:
+      KeyError: if job class is not set for the corresponding name.
+    """
+    job_name = job_class.NAME.lower()
+    if job_name not in cls._job_classes:
+      raise KeyError('job class not set for name: {0:s}'.format(
+          job_class.NAME))
+
+    del cls._job_classes[job_name]
+
+  @classmethod
+  def GetJobInstance(cls, job_name):
+    """Retrieves an instance of a specific job.
+
+    Args:
+      job_name (str): name of the job to retrieve.
+
+    Returns:
+      BaseJob: job instance.
+
+    Raises:
+      KeyError: if job class is not set for the corresponding name.
+    """
+    job_name = job_name.lower()
+    if job_name not in cls._job_classes:
+      raise KeyError(
+          'job class not set for name: {0:s}.'.format(job_name))
+
+    job_class = cls._job_classes[job_name]
+    return job_class()
+
+  @classmethod
+  def GetJobInstances(cls, job_names):
+    """Retrieves instances for all the specified jobs.
+
+    Args:
+      job_names (list[str]): names of the jobs to retrieve.
+
+    Returns:
+      list[BaseJob]: job instances.
+    """
+    job_instances = []
+    for job_name, job_class in iter(cls.GetJobs()):
+      if job_name in job_names:
+        job_instances.append(job_class())
+
+    return job_instances
+
+  @classmethod
+  def GetJobNames(cls):
+    """Retrieves the names of all loaded jobs.
+
+    Returns:
+      list[str]: of job names.
+    """
+    return cls._job_classes.keys()
+
+  @classmethod
+  def GetJobs(cls):
+    """Retrieves the registered jobs.
+
+    Yields:
+      tuple: containing:
+
+        str: the uniquely identifying name of the job
+        type: the job class.
+    """
+    for job_name, job_class in iter(cls._job_classes.items()):
+      yield job_name, job_class
+
+  @classmethod
+  def RegisterJob(cls, job_class):
+    """Registers a job class.
+
+    The job classes are identified by their lower case name.
+
+    Args:
+      job_class (type): the job class to register.
+
+    Raises:
+      KeyError: if job class is already set for the corresponding name.
+    """
+    job_name = job_class.NAME.lower()
+    if job_name in cls._job_classes:
+      raise KeyError('job class already set for name: {0:s}.'.format(
+          job_class.NAME))
+
+    cls._job_classes[job_name] = job_class
+
+  @classmethod
+  def RegisterJobs(cls, job_classes):
+    """Registers a job class.
+
+    The job classes are identified by their lower case name.
+
+    Args:
+      job_classes (list[type]): the job classes to register.
+
+    Raises:
+      KeyError: if job class is already set for the corresponding name.
+    """
+    for job_class in job_classes:
+      cls.RegisterJob(job_class)
diff --git a/turbinia/jobs/plaso.py b/turbinia/jobs/plaso.py
index 647460d..85dc0e9 100644
--- a/turbinia/jobs/plaso.py
+++ b/turbinia/jobs/plaso.py
@@ -21,19 +21,19 @@ from turbinia.evidence import GoogleCloudDisk
 from turbinia.evidence import GoogleCloudDiskRawEmbedded
 from turbinia.evidence import PlasoFile
 from turbinia.evidence import RawDisk
-from turbinia.jobs import TurbiniaJob
+from turbinia.jobs import interface
+from turbinia.jobs import manager
 from turbinia.workers.plaso import PlasoTask
 
 
-class PlasoJob(TurbiniaJob):
-
+class PlasoJob(interface.TurbiniaJob):
+  """Runs Plaso on some evidence to generate a Plaso file."""
   # The types of evidence that this Job will process
   evidence_input = [Directory, RawDisk, GoogleCloudDisk,
                     GoogleCloudDiskRawEmbedded]
   evidence_output = [PlasoFile]
 
-  def __init__(self):
-    super(PlasoJob, self).__init__(name='PlasoJob')
+  NAME = 'PlasoJob'
 
   def create_tasks(self, evidence):
     """Create task for Plaso.
@@ -45,3 +45,6 @@ class PlasoJob(TurbiniaJob):
         A list of PlasoTasks.
     """
     return [PlasoTask() for _ in evidence]
+
+
+manager.JobsManager.RegisterJob(PlasoJob)
diff --git a/turbinia/jobs/psort.py b/turbinia/jobs/psort.py
index ecbf528..eaec4c9 100644
--- a/turbinia/jobs/psort.py
+++ b/turbinia/jobs/psort.py
@@ -19,19 +19,20 @@ from __future__ import unicode_literals
 
 from turbinia.evidence import PlasoFile
 from turbinia.evidence import PlasoCsvFile
-from turbinia.jobs import TurbiniaJob
+from turbinia.jobs import interface
+from turbinia.jobs import manager
 from turbinia.workers.psort import PsortTask
 
 
-class PsortJob(TurbiniaJob):
+class PsortJob(interface.TurbiniaJob):
   """Run psort on PlasoFile to generate a CSV file."""
 
   # The types of evidence that this Job will process
   evidence_input = [PlasoFile]
   evidence_output = [PlasoCsvFile]
 
-  def __init__(self):
-    super(PsortJob, self).__init__(name='PsortJob')
+  NAME = 'PsortJob'
+
 
   def create_tasks(self, evidence):
     """Create task for Psort.
@@ -43,3 +44,6 @@ class PsortJob(TurbiniaJob):
         A list of PsortTasks.
     """
     return [PsortTask() for _ in evidence]
+
+
+manager.JobsManager.RegisterJob(PsortJob)
diff --git a/turbinia/jobs/sshd.py b/turbinia/jobs/sshd.py
index 18b1309..2e5e68b 100644
--- a/turbinia/jobs/sshd.py
+++ b/turbinia/jobs/sshd.py
@@ -24,20 +24,20 @@ from turbinia.evidence import GoogleCloudDiskRawEmbedded
 from turbinia.evidence import ExportedFileArtifact
 from turbinia.evidence import RawDisk
 from turbinia.evidence import ReportText
-from turbinia.jobs import TurbiniaJob
+from turbinia.jobs import interface
+from turbinia.jobs import manager
 
 
-class SSHDExtractionJob(TurbiniaJob):
+class SSHDExtractionJob(interface.TurbiniaJob):
   """Filter input based on regular expression patterns."""
 
   # The types of evidence that this Job will process
   evidence_input = [
-    Directory, RawDisk, GoogleCloudDisk, GoogleCloudDiskRawEmbedded]
+      Directory, RawDisk, GoogleCloudDisk, GoogleCloudDiskRawEmbedded]
 
   evidence_output = [ExportedFileArtifact]
 
-  def __init__(self):
-    super(SSHDExtractionJob, self).__init__(name='SSHDExtractionJob')
+  NAME = 'SSHDExtractionJob'
 
   def create_tasks(self, evidence):
     """Create task.
@@ -49,18 +49,17 @@ class SSHDExtractionJob(TurbiniaJob):
         A list of tasks to schedule.
     """
     tasks = [
-      artifact.FileArtifactExtractionTask('SshdConfigFile') for _ in evidence]
+        artifact.FileArtifactExtractionTask('SshdConfigFile') for _ in evidence]
     return tasks
 
 
-class SSHDAnalysisJob(TurbiniaJob):
+class SSHDAnalysisJob(interface.TurbiniaJob):
   """Filter input based on regular expression patterns."""
 
   evidence_input = [ExportedFileArtifact]
   evidence_output = [ReportText]
 
-  def __init__(self):
-    super(SSHDAnalysisJob, self).__init__(name='SSHDAnalysisJob')
+  NAME = 'SSHDAnalysisJob'
 
   def create_tasks(self, evidence):
     """Create task.
@@ -76,3 +75,6 @@ class SSHDAnalysisJob(TurbiniaJob):
       if evidence_item.artifact_name == 'SshdConfigFile':
         tasks.append(sshd.SSHDAnalysisTask())
     return tasks
+
+
+manager.JobsManager.RegisterJobs([SSHDAnalysisJob, SSHDExtractionJob])
diff --git a/turbinia/jobs/strings.py b/turbinia/jobs/strings.py
index 8c5ca98..1f39e30 100644
--- a/turbinia/jobs/strings.py
+++ b/turbinia/jobs/strings.py
@@ -21,12 +21,13 @@ from turbinia.evidence import GoogleCloudDisk
 from turbinia.evidence import GoogleCloudDiskRawEmbedded
 from turbinia.evidence import RawDisk
 from turbinia.evidence import TextFile
-from turbinia.jobs import TurbiniaJob
+from turbinia.jobs import interface
+from turbinia.jobs import manager
 from turbinia.workers.strings import StringsAsciiTask
 from turbinia.workers.strings import StringsUnicodeTask
 
 
-class StringsJob(TurbiniaJob):
+class StringsJob(interface.TurbiniaJob):
   """Strings collection Job.
 
   This will generate a Unicode and ASCII string collection task for each piece
@@ -37,8 +38,8 @@ class StringsJob(TurbiniaJob):
   evidence_input = [RawDisk, GoogleCloudDisk, GoogleCloudDiskRawEmbedded]
   evidence_output = [TextFile]
 
-  def __init__(self):
-    super(StringsJob, self).__init__(name='StringsJob')
+  NAME = 'StringsJob'
+
 
   def create_tasks(self, evidence):
     """Create task for Strings.
@@ -53,3 +54,6 @@ class StringsJob(TurbiniaJob):
     tasks = [StringsAsciiTask() for _ in evidence]
     tasks.extend([StringsUnicodeTask() for _ in evidence])
     return tasks
+
+
+manager.JobsManager.RegisterJob(StringsJob)
diff --git a/turbinia/jobs/tomcat.py b/turbinia/jobs/tomcat.py
index 7eacd38..dfde693 100644
--- a/turbinia/jobs/tomcat.py
+++ b/turbinia/jobs/tomcat.py
@@ -22,8 +22,11 @@ from turbinia.evidence import GoogleCloudDiskRawEmbedded
 from turbinia.evidence import ExportedFileArtifact
 from turbinia.evidence import RawDisk
 from turbinia.evidence import ReportText
-from turbinia.jobs import TurbiniaJob
-class TomcatExtractionJob(TurbiniaJob):
+from turbinia.jobs import interface
+from turbinia.jobs import manager
+
+
+class TomcatExtractionJob(interface.TurbiniaJob):
   """Extract Apache Tomcat files for analysis."""
 
   # The types of evidence that this Job will process
@@ -32,8 +35,7 @@ class TomcatExtractionJob(TurbiniaJob):
 
   evidence_output = [ExportedFileArtifact]
 
-  def __init__(self):
-    super(TomcatExtractionJob, self).__init__(name='TomcatExtractionJob')
+  NAME = 'TomcatExtractionJob'
 
   def create_tasks(self, evidence):
     """Create task.
@@ -48,14 +50,14 @@ class TomcatExtractionJob(TurbiniaJob):
         artifact.FileArtifactExtractionTask('TomcatFiles') for _ in evidence]
     return tasks
 
-class TomcatAnalysisJob(TurbiniaJob):
+
+class TomcatAnalysisJob(interface.TurbiniaJob):
   """Create tasks to analyse Apache Tomcat files."""
 
   evidence_input = [ExportedFileArtifact]
   evidence_output = [ReportText]
 
-  def __init__(self):
-    super(TomcatAnalysisJob, self).__init__(name='TomcatAnalysisJob')
+  NAME = 'TomcatAnalysisJob'
 
   def create_tasks(self, evidence):
     """Create task.
@@ -72,3 +74,6 @@ class TomcatAnalysisJob(TurbiniaJob):
       if evidence_item.artifact_name == 'TomcatFile':
         tasks.append(tomcat.TomcatAnalysisTask())
     return tasks
+
+
+manager.JobsManager.RegisterJobs([TomcatExtractionJob, TomcatAnalysisJob])
diff --git a/turbinia/jobs/worker_stat.py b/turbinia/jobs/worker_stat.py
index 81a5aab..bdba1df 100644
--- a/turbinia/jobs/worker_stat.py
+++ b/turbinia/jobs/worker_stat.py
@@ -20,19 +20,19 @@ from __future__ import unicode_literals
 from turbinia.evidence import Directory
 from turbinia.evidence import RawDisk
 from turbinia.evidence import ReportText
-from turbinia.jobs import TurbiniaJob
+from turbinia.jobs import interface
+from turbinia.jobs import manager
 from turbinia.workers.worker_stat import StatTask
 
 
-class StatJob(TurbiniaJob):
+class StatJob(interface.TurbiniaJob):
   """Job to run Stat."""
 
   # The types of evidence that this Job will process
   evidence_input = [RawDisk, Directory]
   evidence_output = [ReportText]
 
-  def __init__(self):
-    super(StatJob, self).__init__(name='StatJob')
+  NAME = 'StatJob'
 
   def create_tasks(self, evidence):
     """Create task for Stat.
@@ -44,3 +44,6 @@ class StatJob(TurbiniaJob):
         A list of StatTasks.
     """
     return [StatTask() for _ in evidence]
+
+
+manager.JobsManager.RegisterJob(StatJob)
diff --git a/turbinia/processors/mount_local.py b/turbinia/processors/mount_local.py
index 44b3060..91b621b 100644
--- a/turbinia/processors/mount_local.py
+++ b/turbinia/processors/mount_local.py
@@ -51,15 +51,38 @@ def PreprocessMountDisk(evidence):
 
   evidence.mount_path = tempfile.mkdtemp(prefix='turbinia', dir=mount_prefix)
 
-  if hasattr(evidence, 'mount_partition') and evidence.mount_partition:
-    src_path = '{0:s}-part{1:d}'.format(
-        evidence.local_path, evidence.mount_partition)
-  else:
-    src_path = evidence.local_path
+  # Two steps: first use losetup to parse the RawDisk potential partition table
+  # and make block devices per partitions.
+  # Then we mount the partition provideed, or the first one we find.
 
   # TODO(aarontp): Remove hard-coded sudo in commands:
   # https://github.com/google/turbinia/issues/73
-  mount_cmd = ['sudo', 'mount', src_path, evidence.mount_path]
+  losetup_command = [
+      'sudo', 'losetup', '--show', '--find', '-P', evidence.local_path]
+  log.info('Running command {0:s}'.format(' '.join(losetup_command)))
+  try:
+    losetup_device = subprocess.check_output(losetup_command)
+  except subprocess.CalledProcessError as e:
+    raise TurbiniaException('Could not set losetup devices {0!s}'.format(e))
+  evidence.losetup_device = losetup_device.strip()
+
+  # Here, evidence.losetup_device is something like '/dev/loopX'.
+
+  if not evidence.mount_partition:
+    # The first partition loop-device made by losetup is loopXp1
+    evidence.mount_partition = 1
+
+  path_to_partition = '{0:s}p{1:d}'.format(
+      evidence.losetup_device, evidence.mount_partition)
+  # If evidence.local_path points to something with a partition table, we have
+  # block devices at /dev/loopXpY.
+  if not os.path.exists(path_to_partition):
+    log.info('Could not find {0:s}, trying {1:s}'.format(
+        path_to_partition, evidence.losetup_device))
+    # Else, we only have /dev/loopX
+    path_to_partition = evidence.losetup_device
+
+  mount_cmd = ['sudo', 'mount', path_to_partition, evidence.mount_path]
   log.info('Running: {0:s}'.format(' '.join(mount_cmd)))
   try:
     subprocess.check_call(mount_cmd)
@@ -82,6 +105,13 @@ def PostprocessUnmountDisk(evidence):
   except subprocess.CalledProcessError as e:
     raise TurbiniaException('Could not unmount directory {0!s}'.format(e))
 
+  losetup_cmd = ['sudo', 'losetup', '-d', evidence.losetup_device]
+  log.info('Running: {0:s}'.format(' '.join(losetup_cmd)))
+  try:
+    subprocess.check_call(losetup_cmd)
+  except subprocess.CalledProcessError as e:
+    raise TurbiniaException('Could not delete losetup device {0!s}'.format(e))
+
   log.info('Removing mount path {0:s}'.format(evidence.mount_path))
   try:
     os.rmdir(evidence.mount_path)
diff --git a/turbinia/task_manager.py b/turbinia/task_manager.py
index 6f89820..263651a 100644
--- a/turbinia/task_manager.py
+++ b/turbinia/task_manager.py
@@ -22,8 +22,8 @@ import time
 import turbinia
 from turbinia import evidence
 from turbinia import config
-from turbinia import jobs
 from turbinia import state_manager
+from turbinia.jobs import manager as jobs_manager
 
 config.LoadConfig()
 if config.TASK_MANAGER == 'PSQ':
@@ -104,7 +104,7 @@ class BaseTaskManager(object):
     """Does setup of Task manager and its dependencies."""
     self._backend_setup(*args, **kwargs)
     # TODO(aarontp): Consider instantiating a job per evidence object
-    self.jobs = jobs.get_jobs()
+    self.jobs = jobs_manager.JobsManager.GetJobs()
 
   def add_evidence(self, evidence_):
     """Adds new evidence and creates tasks to process it.
diff --git a/turbinia/turbiniactl.py b/turbinia/turbiniactl.py
index 5229ca4..56dddc0 100755
--- a/turbinia/turbiniactl.py
+++ b/turbinia/turbiniactl.py
@@ -138,7 +138,7 @@ def main():
   parser_rawdisk.add_argument(
       '-P',
       '--mount_partition',
-      default=0,
+      default=1,
       type=int,
       help='The partition number to use when mounting this disk.  Defaults to '
       'the entire raw disk.  Only affects mounting, and not what gets '
diff --git a/turbinia/workers/__init__.py b/turbinia/workers/__init__.py
index 30ae8da..00e2926 100644
--- a/turbinia/workers/__init__.py
+++ b/turbinia/workers/__init__.py
@@ -120,7 +120,8 @@ class TurbiniaTaskResult(object):
     for evidence in self.evidence:
       if evidence.local_path:
         self.saved_paths.append(evidence.local_path)
-        task.output_manager.save_evidence(evidence, self)
+        if evidence.copyable and not config.SHARED_FILESYSTEM:
+          task.output_manager.save_evidence(evidence, self)
       if not evidence.request_id:
         evidence.request_id = self.request_id
 


Test patch:
diff --git a/turbinia/jobs/imports_test.py b/turbinia/jobs/imports_test.py
new file mode 100644
index 0000000..4ed2e40
--- /dev/null
+++ b/turbinia/jobs/imports_test.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# Copyright 2015 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# -*- coding: utf-8 -*-
+"""Tests that all jobs are imported correctly."""
+
+from __future__ import unicode_literals
+
+import io
+import re
+import os
+import unittest
+
+
+class JobImportTest(unittest.TestCase):
+  """Tests that job classes are imported correctly."""
+
+  _JOBS_PATH = os.path.abspath(os.path.dirname(__file__))
+
+  _IGNORABLE_FILES = frozenset(['manager.py', 'interface.py'])
+
+  _FILENAME_REGEXP = re.compile(r'^[^_]\w+\.py$')
+
+  def _AssertFilesImportedInInit(self, path, ignorable_files):
+    """Checks that files in path are imported in __init__.py
+
+    Args:
+      path (str): path to directory containing an __init__.py file and other
+          Python files which should be imported.
+      ignorable_files (Iterable[str]): names of Python files that don't need to
+          appear in __init__.py. For example, 'manager.py'.
+    """
+    init_path = '{0:s}/__init__.py'.format(path)
+    with io.open(init_path, mode='r', encoding='utf-8') as init_file:
+      init_content = init_file.read()
+
+    for file_path in os.listdir(path):
+      filename = os.path.basename(file_path)
+      if filename in ignorable_files:
+        continue
+      if filename.endswith('_test.py'):
+        continue
+      if self._FILENAME_REGEXP.search(filename):
+        module_name, _, _ = filename.partition('.')
+        import_expression = re.compile(r' import {0:s}\b'.format(module_name))
+
+        self.assertRegexpMatches(
+            init_content, import_expression,
+            '{0:s} not imported in {1:s}'.format(module_name, init_path))
+
+  def testJobsImported(self):
+    """Tests that all jobs are imported."""
+    self._AssertFilesImportedInInit(self._JOBS_PATH, self._IGNORABLE_FILES)
+
+
+if __name__ == '__main__':
+  unittest.main()
diff --git a/turbinia/jobs/manager_test.py b/turbinia/jobs/manager_test.py
new file mode 100644
index 0000000..b613f2a
--- /dev/null
+++ b/turbinia/jobs/manager_test.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""Tests for the job manager."""
+
+from __future__ import unicode_literals
+
+import unittest
+
+from turbinia.jobs import interface
+from turbinia.jobs import manager
+
+
+class TestJob1(interface.TurbiniaJob):
+  """Test job."""
+
+  NAME = 'testjob1'
+
+  # pylint: disable=unused-argument
+  def create_tasks(self, evidence):
+    """Returns None, for testing."""
+    return None
+
+class TestJob2(interface.TurbiniaJob):
+  """Test job."""
+
+  NAME = 'testjob2'
+
+  # pylint: disable=unused-argument
+  def create_tasks(self, evidence):
+    """Returns None, for testing."""
+    return None
+
+class JobsManagerTest(unittest.TestCase):
+  """Tests for the jobs manager."""
+
+  # pylint: disable=protected-access
+
+  def tearDown(self):
+    """Cleans up after running an individual test."""
+    # Deregister the test jobs if the test failed.
+    try:
+      manager.JobsManager.DeregisterJob(TestJob1)
+    except KeyError:
+      pass
+    try:
+      manager.JobsManager.DeregisterJob(TestJob2)
+    except KeyError:
+      pass
+
+  def testJobRegistration(self):
+    """Tests the registration and deregistration of jobs."""
+    number_of_jobs = len(manager.JobsManager._job_classes)
+    manager.JobsManager.RegisterJob(TestJob1)
+    self.assertEqual(
+        number_of_jobs + 1, len(
+            manager.JobsManager._job_classes))
+
+    with self.assertRaises(KeyError):
+      manager.JobsManager.RegisterJob(TestJob1)
+
+    manager.JobsManager.DeregisterJob(TestJob1)
+
+    self.assertEqual(
+        number_of_jobs, len(manager.JobsManager._job_classes))
+
+    number_of_jobs = len(manager.JobsManager._job_classes)
+    manager.JobsManager.RegisterJobs([TestJob1, TestJob2])
+    self.assertEqual(
+        number_of_jobs + 2, len(
+            manager.JobsManager._job_classes))
+
+    with self.assertRaises(KeyError):
+      manager.JobsManager.RegisterJob(TestJob1)
+
+    manager.JobsManager.DeregisterJob(TestJob1)
+    manager.JobsManager.DeregisterJob(TestJob2)
+
+    self.assertEqual(
+        number_of_jobs, len(manager.JobsManager._job_classes))
+
+
+  def testGetJobInstance(self):
+    """Tests the GetJobInstance function."""
+    manager.JobsManager.RegisterJob(TestJob1)
+    job = manager.JobsManager.GetJobInstance('testjob1')
+    self.assertIsNotNone(job)
+    self.assertEqual(job.NAME, 'testjob1')
+
+    with self.assertRaises(KeyError):
+      manager.JobsManager.GetJobInstance('bogus')
+    manager.JobsManager.DeregisterJob(TestJob1)
+
+  def testGetJobInstances(self):
+    """Tests getting job objects by name."""
+    manager.JobsManager.RegisterJob(TestJob1)
+    job_names = manager.JobsManager.GetJobNames()
+    jobs = manager.JobsManager.GetJobInstances(job_names)
+    self.assertEqual(len(job_names), len(jobs))
+    for job in jobs:
+      self.assertIsInstance(job, interface.TurbiniaJob)
+
+
+if __name__ == '__main__':
+  unittest.main()