Repository: dbt-labs/dbt-core

Issue:
Add data processed info into dbt run logs for all statement types
### Describe the feature
When using BigQuery `dbt run` logs should show the data scan value for create table statement and not just for script.

### Describe alternatives you've considered
No alternative

### Additional context
This specific improvement is about BigQuery. However, this might apply for other database too I do not have experience to give specific examples.

### Who will this benefit?
Everyone using dbt to get easier access to usage.


Discussion:
Very cool idea!

The code for printing out bytes billed is fairly straightforward:
https://github.com/fishtown-analytics/dbt/blob/c9b34681b756ccd4b5c4f0a3bd8f12f48f23081d/plugins/bigquery/dbt/adapters/bigquery/connections.py#L250-L252

I think it would just be a matter of reporting the `total_bytes_processed` for other `query_job` statement types.

@alepuccetti Is this something you'd be interested in working on? I think this could be a great opportunity for a first contribution :)
@jtcohen6  Yes, I would be interested. You can assigned to me if this is the procedure.
It's all yours! Happy to help if you want more direction or get stuck

Resolving patch:
diff --git a/core/dbt/utils.py b/core/dbt/utils.py
--- a/core/dbt/utils.py
+++ b/core/dbt/utils.py
@@ -510,12 +510,23 @@ def __get__(self, obj, objtype):
 
 
 def format_bytes(num_bytes):
-    for unit in ['Bytes', 'KB', 'MB', 'GB', 'TB']:
+    for unit in ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB']:
         if abs(num_bytes) < 1024.0:
             return f"{num_bytes:3.1f} {unit}"
         num_bytes /= 1024.0
 
-    return "> 1024 TB"
+    num_bytes *= 1024.0
+    return f"{num_bytes:3.1f} {unit}"
+
+
+def format_rows_number(rows_number):
+    for unit in ['', 'k', 'm', 'b', 't']:
+        if abs(rows_number) < 1000.0:
+            return f"{rows_number:3.1f}{unit}".strip()
+        rows_number /= 1000.0
+
+    rows_number *= 1000.0
+    return f"{rows_number:3.1f}{unit}".strip()
 
 
 # a little concurrent.futures.Executor for single-threaded mode
diff --git a/plugins/bigquery/dbt/adapters/bigquery/connections.py b/plugins/bigquery/dbt/adapters/bigquery/connections.py
--- a/plugins/bigquery/dbt/adapters/bigquery/connections.py
+++ b/plugins/bigquery/dbt/adapters/bigquery/connections.py
@@ -8,7 +8,7 @@
 from google.api_core import retry, client_info
 from google.oauth2 import service_account
 
-from dbt.utils import format_bytes
+from dbt.utils import format_bytes, format_rows_number
 from dbt.clients import agate_helper, gcloud
 from dbt.exceptions import (
     FailedToConnectException, RuntimeException, DatabaseException
@@ -245,16 +245,21 @@ def execute(self, sql, auto_begin=False, fetch=None):
             conn = self.get_thread_connection()
             client = conn.handle
             table = client.get_table(query_job.destination)
-            status = 'CREATE TABLE ({})'.format(table.num_rows)
+            processed = format_bytes(query_job.total_bytes_processed)
+            status = 'CREATE TABLE ({} rows, {} processed)'.format(
+                format_rows_number(table.num_rows),
+                format_bytes(query_job.total_bytes_processed),
+            )
 
         elif query_job.statement_type == 'SCRIPT':
             processed = format_bytes(query_job.total_bytes_processed)
             status = f'SCRIPT ({processed} processed)'
 
         elif query_job.statement_type in ['INSERT', 'DELETE', 'MERGE']:
-            status = '{} ({})'.format(
+            status = '{} ({} rows, {} processed)'.format(
                 query_job.statement_type,
-                query_job.num_dml_affected_rows
+                format_rows_number(query_job.num_dml_affected_rows),
+                format_bytes(query_job.total_bytes_processed),
             )
 
         else:


Test patch:
diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py
--- a/test/unit/test_utils.py
+++ b/test/unit/test_utils.py
@@ -93,7 +93,6 @@ def test__simple_cases(self):
         actual = dbt.utils.deep_map(self.intify_all, expected)
         self.assertEqual(actual, expected)
 
-
     @staticmethod
     def special_keypath(value, keypath):
 
@@ -150,7 +149,25 @@ def test__simple_cases(self):
         self.assertEqual(dbt.utils.format_bytes(1024**2*1.5), '1.5 MB')
         self.assertEqual(dbt.utils.format_bytes(1024**3*52.6), '52.6 GB')
         self.assertEqual(dbt.utils.format_bytes(1024**4*128), '128.0 TB')
-        self.assertEqual(dbt.utils.format_bytes(1024**5+1), '> 1024 TB')
+        self.assertEqual(dbt.utils.format_bytes(1024**5), '1.0 PB')
+        self.assertEqual(dbt.utils.format_bytes(1024**5*31.4), '31.4 PB')
+        self.assertEqual(dbt.utils.format_bytes(1024**6), '1024.0 PB')
+        self.assertEqual(dbt.utils.format_bytes(1024**6*42), '43008.0 PB')
+
+
+class TestRowsNumberFormatting(unittest.TestCase):
+
+    def test__simple_cases(self):
+        self.assertEqual(dbt.utils.format_rows_number(-1), '-1.0')
+        self.assertEqual(dbt.utils.format_rows_number(0), '0.0')
+        self.assertEqual(dbt.utils.format_rows_number(20), '20.0')
+        self.assertEqual(dbt.utils.format_rows_number(1030), '1.0k')
+        self.assertEqual(dbt.utils.format_rows_number(1000**2*1.5), '1.5m')
+        self.assertEqual(dbt.utils.format_rows_number(1000**3*52.6), '52.6b')
+        self.assertEqual(dbt.utils.format_rows_number(1000**3*128), '128.0b')
+        self.assertEqual(dbt.utils.format_rows_number(1000**4), '1.0t')
+        self.assertEqual(dbt.utils.format_rows_number(1000**4*31.4), '31.4t')
+        self.assertEqual(dbt.utils.format_rows_number(1000**5*31.4), '31400.0t')  # noqa: E501
 
 
 class TestMultiDict(unittest.TestCase):