Repository: google/turbinia
Pull request #431 (2019-08-23 23:18:30)

Add saving of metadata.json files to output manager
For evidence objects with `save_metadata` set, this will write a `.metadata.json` file with all of the evidence context written to it.


Patch:
diff --git a/terraform/gcp/machines.tf b/terraform/gcp/machines.tf
deleted file mode 100644
index 5fc6a93..0000000
--- a/terraform/gcp/machines.tf
+++ /dev/null
@@ -1,90 +0,0 @@
-resource "google_compute_instance" "turbinia-server" {
-  name         = "turbinia-server"
-  machine_type = "g1-small"
-  zone         = "${var.zone}"
-
-  # Allow to stop/start the machine to enable change machine type.
-  allow_stopping_for_update = true
-
-  # Use default Ubuntu image as operating system.
-  boot_disk {
-    initialize_params {
-      image = "${var.ubuntu_image}"
-    }
-  }
-
-  # Assign a generated public IP address. Needed for SSH access.
-  network_interface {
-    network       = "default"
-    access_config = {}
-  }
-
-  # Provision the machine with a script.
-  metadata_startup_script = "${data.template_file.turbinia-server.rendered}"
-}
-
-resource "google_compute_instance" "turbinia-worker" {
-  name         = "turbinia-worker"
-  machine_type = "g1-small"
-  zone         = "${var.zone}"
-
-  # How many workers to start
-  count = 3
-
-  # Allow to stop/start the machine to enable change machine type.
-  allow_stopping_for_update = true
-
-  # Use default Ubuntu image as operating system.
-  boot_disk {
-    initialize_params {
-      image = "${var.ubuntu_image}"
-    }
-  }
-
-  # Assign a generated public IP address. Needed for SSH access.
-  network_interface {
-    network       = "default"
-    access_config = {}
-  }
-
-  # Provision the machine with a script.
-  metadata_startup_script = "${data.template_file.turbinia-worker.rendered}"
-}
-
-# Enabling GCP services individually because the google_project_services
-# resource type will disable services not in the list.
-resource "google_project_service" "project" {
-  project = "${var.project}"
-  service = "cloudfunctions.googleapis.com"
-  disable_on_destroy = false
-}
-
-resource "google_project_service" "project" {
-  project = "${var.project}"
-  service = "compute.googleapis.com"
-  disable_on_destroy = false
-}
-
-resource "google_project_service" "project" {
-  project = "${var.project}"
-  service = "datastore.googleapis.com"
-  disable_on_destroy = false
-}
-
-resource "google_project_service" "project" {
-  project = "${var.project}"
-  service = "iam.googleapis.com"
-  disable_on_destroy = false
-}
-
-resource "google_project_service" "project" {
-  project = "${var.project}"
-  service = "pubsub.googleapis.com"
-  disable_on_destroy = false
-}
-
-resource "google_project_service" "project" {
-  project = "${var.project}"
-  service = "storage-component.googleapis.com"
-  disable_on_destroy = false
-}
diff --git a/terraform/gcp/main.tf b/terraform/gcp/main.tf
deleted file mode 100644
index 13fcca6..0000000
--- a/terraform/gcp/main.tf
+++ /dev/null
@@ -1,20 +0,0 @@
-terraform {
-  # Use local state storafe by default. For production environments please
-  # consider  using a more robust backend.
-  backend "local" {
-    path = "terraform.tfstate"
-  }
-
-  /*
-  # Use Google Cloud Storage for robust, collaborative state storage.
-  # Note: The bucket name need to be globally unique.
-  backend "gcs" {
-    bucket = "GLOBALY UNIQUE BUCKET NAME "
-  }
-  */
-}
-
-provider "google" {
-  project = "${var.project}"
-  region  = "${var.region}"
-}
diff --git a/terraform/gcp/startup_scripts/install_server.sh b/terraform/gcp/startup_scripts/install_server.sh
deleted file mode 100644
index b604a92..0000000
--- a/terraform/gcp/startup_scripts/install_server.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-
-
-# --- BEFORE MAIN (DO NOT EDIT) ---
-
-# Exit on any error
-set -e
-
-# Default constants.
-readonly BOOT_FINISHED_FILE="/var/lib/cloud/instance/boot-finished"
-readonly STARTUP_FINISHED_FILE="/var/lib/cloud/instance/startup-script-finished"
-
-# Redirect stdout and stderr to logfile
-exec > /var/log/terraform_provision.log
-exec 2>&1
-
-# Exit if the startup script has already been executed successfully
-if [[ -f "$${STARTUP_FINISHED_FILE}" ]]; then
-  exit 0
-fi
-
-# Wait for cloud-init to finish all tasks
-until [[ -f "$${BOOT_FINISHED_FILE}" ]]; do
-  sleep 1
-done
-
-# --- END BEFORE MAIN ---
-
-
-# --- MAIN ---
-
-# TODO
-
-# --- END MAIN ---
-
-
-# --- AFTER MAIN (DO NOT EDIT)
-
-date > "$${STARTUP_FINISHED_FILE}"
-echo "Startup script finished successfully"
-
-# --- END AFTER MAIN ---
\ No newline at end of file
diff --git a/terraform/gcp/startup_scripts/install_worker.sh b/terraform/gcp/startup_scripts/install_worker.sh
deleted file mode 100644
index b604a92..0000000
--- a/terraform/gcp/startup_scripts/install_worker.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-
-
-# --- BEFORE MAIN (DO NOT EDIT) ---
-
-# Exit on any error
-set -e
-
-# Default constants.
-readonly BOOT_FINISHED_FILE="/var/lib/cloud/instance/boot-finished"
-readonly STARTUP_FINISHED_FILE="/var/lib/cloud/instance/startup-script-finished"
-
-# Redirect stdout and stderr to logfile
-exec > /var/log/terraform_provision.log
-exec 2>&1
-
-# Exit if the startup script has already been executed successfully
-if [[ -f "$${STARTUP_FINISHED_FILE}" ]]; then
-  exit 0
-fi
-
-# Wait for cloud-init to finish all tasks
-until [[ -f "$${BOOT_FINISHED_FILE}" ]]; do
-  sleep 1
-done
-
-# --- END BEFORE MAIN ---
-
-
-# --- MAIN ---
-
-# TODO
-
-# --- END MAIN ---
-
-
-# --- AFTER MAIN (DO NOT EDIT)
-
-date > "$${STARTUP_FINISHED_FILE}"
-echo "Startup script finished successfully"
-
-# --- END AFTER MAIN ---
\ No newline at end of file
diff --git a/terraform/gcp/templates.tf b/terraform/gcp/templates.tf
deleted file mode 100644
index 1925175..0000000
--- a/terraform/gcp/templates.tf
+++ /dev/null
@@ -1,12 +0,0 @@
-# Register startup scripts. These scripts will be rendered and used when
-# machines boot up.
-
-data "template_file" "server" {
-  template = "${file("${path.module}/startup_scripts/install_server.sh")}"
-  vars {}
-}
-
-data "template_file" "client" {
-  template = "${file("${path.module}/startup_scripts/install_client.sh")}"
-  vars {}
-}
diff --git a/terraform/gcp/variables.tf b/terraform/gcp/variables.tf
deleted file mode 100644
index bb008db..0000000
--- a/terraform/gcp/variables.tf
+++ /dev/null
@@ -1,4 +0,0 @@
-variable "project"            {}
-variable "region"             {}
-variable "zone"               {}
-variable "ubuntu_image"  { default = "ubuntu-os-cloud/ubuntu-1804-lts" }
diff --git a/turbinia/evidence.py b/turbinia/evidence.py
index 61a5bfc..6dae495 100644
--- a/turbinia/evidence.py
+++ b/turbinia/evidence.py
@@ -77,27 +77,31 @@ class Evidence(object):
   Attributes:
     config (dict): Configuration options from the request to be used when
         processing this evidence.
-    cloud_only: Set to True for evidence types that can only be processed in a
-        cloud environment, e.g. GoogleCloudDisk.
-    context_dependent: Whether this evidence is required to be built upon the
-        context of a parent evidence.
-    copyable: Whether this evidence can be copied.  This will be set to True for
-        object types that we want to copy to/from storage (e.g. PlasoFile, but
-        not RawDisk).
-    name: Name of evidence.
-    description: Description of evidence.
-    saved_path (string): Path to secondary location evidence is saved for later
+    cloud_only (bool): Set to True for evidence types that can only be processed
+        in a cloud environment, e.g. GoogleCloudDisk.
+    context_dependent (bool): Whether this evidence is required to be built upon
+        the context of a parent evidence.
+    copyable (bool): Whether this evidence can be copied.  This will be set to
+        True for object types that we want to copy to/from storage (e.g.
+        PlasoFile, but not RawDisk).
+    name (str): Name of evidence.
+    description (str): Description of evidence.
+    saved_path (str): Path to secondary location evidence is saved for later
         retrieval (e.g. GCS).
-    saved_path_type (string): The name of the output writer that saved evidence
+    saved_path_type (str): The name of the output writer that saved evidence
         to the saved_path location.
-    source: String indicating where evidence came from (including tool version
-        that created it, if appropriate).
-    local_path: A string of the local_path to the evidence.
-    tags: dict of extra tags associated with this evidence.
-    request_id: The id of the request this evidence came from, if any.
-    parent_evidence: The Evidence object that was used to generate this one, and
-        which pre/post process methods we need to re-execute to access data
-        relevant to us.
+    source (str): String indicating where evidence came from (including tool
+        version that created it, if appropriate).
+    local_path (str): A string of the local_path to the evidence.
+    tags (dict): Extra tags associated with this evidence.
+    request_id (str): The id of the request this evidence came from, if any.
+    parent_evidence (Evidence): The Evidence object that was used to generate
+        this one, and which pre/post process methods we need to re-execute to
+        access data relevant to us.
+    save_metadata (bool): Evidence with this property set will save a metadata
+        file alongside the Evidence when saving to external storage.  The
+        metadata file will contain all of the key=value pairs sent along with
+        the processing request in the recipe.  The output is in JSON format
   """
 
   # The list of attributes a given piece of Evidence requires to be set
@@ -117,6 +121,7 @@ class Evidence(object):
     self.tags = tags if tags else {}
     self.request_id = request_id
     self.parent_evidence = None
+    self.save_metadata = False
 
     # List of jobs that have processed this evidence
     self.processed_by = []
@@ -400,6 +405,7 @@ class PlasoFile(Evidence):
     self.plaso_version = plaso_version
     super(PlasoFile, self).__init__(*args, **kwargs)
     self.copyable = True
+    self.save_metadata = True
 
 
 class PlasoCsvFile(PlasoFile):
@@ -409,6 +415,7 @@ class PlasoCsvFile(PlasoFile):
     """Initialization for Plaso File evidence."""
     self.plaso_version = plaso_version
     super(PlasoCsvFile, self).__init__(*args, **kwargs)
+    self.save_metadata = False
 
 
 # TODO(aarontp): Find a way to integrate this into TurbiniaTaskResult instead.
diff --git a/turbinia/output_manager.py b/turbinia/output_manager.py
index 0c1c9e4..768b258 100644
--- a/turbinia/output_manager.py
+++ b/turbinia/output_manager.py
@@ -17,6 +17,7 @@
 from __future__ import unicode_literals
 
 import errno
+import json
 import logging
 import os
 import re
@@ -133,9 +134,34 @@ class OutputManager(object):
 
     Returns:
       An evidence object
+
+    Raises:
+      TurbiniaException: If serialization or writing of evidence config fails
     """
     path, path_type, local_path = self.save_local_file(
         evidence_.local_path, result)
+
+    if evidence_.save_metadata:
+      metadata = evidence_.config.copy()
+      metadata_path = '{0:s}.metadata.json'.format(local_path)
+      try:
+        json_str = json.dumps(metadata)
+      except TypeError as exception:
+        raise TurbiniaException(
+            'Could not serialize Evidence config for {0:s}: {1:s}'.format(
+                evidence_.name, exception))
+
+      try:
+        log.debug('Writing metadata file to {0:s}'.format(metadata_path))
+        with open(metadata_path, 'wb') as file_handle:
+          file_handle.write(json_str.encode('utf-8'))
+      except IOError as exception:
+        raise TurbiniaException(
+            'Could not write metadata file {0:s}: {1:s}'.format(
+                metadata_path, exception))
+
+      self.save_local_file(metadata_path, result)
+
     # Set the evidence local_path from the saved path info so that in cases
     # where tasks are saving evidence into the temp dir, we'll get the newly
     # copied version from the saved output path.
@@ -168,7 +194,7 @@ class OutputManager(object):
     local_path = None
     for writer in self._output_writers:
       new_path = writer.copy_to(file_)
-      if new_path:
+      if new_path and result:
         result.saved_paths.append(new_path)
         saved_path = new_path
         saved_path_type = writer.name


Test patch:
diff --git a/turbinia/output_manager_test.py b/turbinia/output_manager_test.py
index caec1e8..73613b6 100644
--- a/turbinia/output_manager_test.py
+++ b/turbinia/output_manager_test.py
@@ -124,6 +124,40 @@ class TestLocalOutputManager(unittest.TestCase):
     self.assertTrue(os.path.exists(dst_file))
     self.assertIsInstance(return_evidence, evidence.Evidence)
     self.assertIn(dst_file, return_evidence.saved_path)
+    # Makes sure evidence without save_metadata set does not generate a
+    # metadata file
+    self.assertFalse(os.path.exists('{0:s}.metadata.json'.format(dst_file)))
+
+  def testSaveEvidenceWithMetadata(self):
+    """Test the save_evidence method with metadata file."""
+    # Set path to None so we don't try to initialize GCS outout writer.
+    config.GCS_OUTPUT_PATH = None
+    self.task.output_manager.setup(self.task)
+    tmp_dir, local_dir = self.task.output_manager.get_local_output_dirs()
+    self.task.result = mock.MagicMock()
+    self.task.result.saved_paths = []
+    test_contents = 'test_contents'
+    test_file = 'test-file.out'
+    src_file = os.path.join(tmp_dir, test_file)
+    dst_file = os.path.join(local_dir, test_file)
+    test_evidence = evidence.Evidence()
+    test_evidence.save_metadata = True
+    test_evidence.config = {'foo': 'bar'}
+
+    with open(src_file, 'w') as fh:
+      fh.write(test_contents)
+    test_evidence.local_path = src_file
+
+    self.assertFalse(os.path.exists(dst_file))
+    return_evidence = self.task.output_manager.save_evidence(
+        test_evidence, self.task.result)
+    self.assertTrue(os.path.exists(dst_file))
+    self.assertIsInstance(return_evidence, evidence.Evidence)
+    self.assertIn(dst_file, return_evidence.saved_path)
+    metadata_file = '{0:s}.metadata.json'.format(dst_file)
+    metadata_contents = b'{"foo": "bar"}'
+    self.assertTrue(os.path.exists(metadata_file))
+    self.assertEqual(open(metadata_file, 'rb').read(), metadata_contents)
 
 
 class TestLocalOutputWriter(unittest.TestCase):