Repository: GoogleCloudPlatform/professional-services

Issue:
examples/billboard/billboard.py - Permission Denied
Hi, 
Created export standard and detailed billing information to bq.  Run the code from cloudshell. I have I followed the tutorial from bit.ly/billboard-template and I see an error:
`Permission Denied so you do not have project level permission or provided wrong project id, please check.`

I have roles assigned:
- Billing Administrator
- Owner
- Security Admin
- BigQuery Admin
- Organization Administrator

I tested it in two different gcp organizations and the result is the same. project_id get from PROJECT_ID env and this is the project where I run the python and where is bq with data.

set of my libs:
`cachetools==5.2.0
certifi==2022.9.24
charset-normalizer==2.1.1
colorama==0.4.6
google-api-core==2.10.2
google-auth==2.14.1
google-cloud-bigquery==3.4.0
google-cloud-bigquery-storage==2.16.2
google-cloud-billing==1.7.3
google-cloud-core==2.3.2
google-crc32c==1.5.0
google-resumable-media==2.4.0
googleapis-common-protos==1.57.0
grpc-google-iam-v1==0.12.4
grpcio==1.50.0
grpcio-status==1.50.0
idna==3.4
numpy==1.23.5
packaging==21.3
proto-plus==1.22.1
protobuf==4.21.9
pyarrow==10.0.1
pyasn1==0.4.8
pyasn1-modules==0.2.8
pyparsing==3.0.9
python-dateutil==2.8.2
requests==2.28.1
rsa==4.9
six==1.16.0
urllib3==1.26.13`


Discussion:
Hi,

I replicated the issue. 

To solve it, you need to enable "Cloud Billing API" 

@ayapsone, I would like to suggest an improvement, to avoid this issue. 
In this block:

``
    project_id_temp = "projects/{}".format(args.PROJECT_ID)
    try:
        project_billing_info = billing.CloudBillingClient(
        ).get_project_billing_info(name=project_id_temp)
    except PermissionDenied:
        print(
            "Permission Denied so you do not have project level permission or provided wrong project id, please check."
        )
        return sys.exit(1)
``

Add a validation to check if the API is enabled, because returning "Permissions Denied" is hiding the root cause.

Thank you.


@ayapsone Do you have any input on the Jan 11 comment?
I have a new code to be pushed and will try to incorporate this.

Resolving patch:
diff --git a/examples/billboard/billboard.py b/examples/billboard/billboard.py
--- a/examples/billboard/billboard.py
+++ b/examples/billboard/billboard.py
@@ -12,8 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# from googleapiclient import discovery
 from google.cloud import bigquery
 from google.cloud import billing
+from google.api_core import client_info as http_client_info
 from google.api_core.exceptions import PermissionDenied
 from google.cloud.exceptions import NotFound
 import argparse
@@ -21,14 +23,13 @@
 from colorama import Back
 from colorama import Style
 
-bq_client = bigquery.Client()
-
 base_url = "https://lookerstudio.google.com/reporting/create?"
 report_part_url = base_url + "c.reportId=64387229-05e0-4951-aa3f-e7349bbafc07"
 report_base_url = report_part_url + "&r.reportName=MyBillboard"
+report_base_url = report_base_url + "&ds.ds8.refreshFields=false"
 
-std_proj_url = "&ds.ds39.connector=bigQuery&ds.ds39.projectId={}"
-std_table_url = "&ds.ds39.type=TABLE&ds.ds39.datasetId={}&ds.ds39.tableId={}"
+std_proj_url = "&ds.ds8.connector=bigQuery&ds.ds8.projectId={}"
+std_table_url = "&ds.ds8.type=TABLE&ds.ds8.datasetId={}&ds.ds8.tableId={}"
 standard_view_url = std_proj_url + std_table_url
 
 dtl_proj_url = "&ds.ds93.connector=bigQuery&ds.ds93.projectId={}"
@@ -39,7 +40,18 @@
 isDetailedExportDifferentLocation = False
 detailedBBDataset = ""
 
-app_version = "2.0"
+app_version = "3.0"
+
+APPLICATION_NAME = "professional-services/billboard"
+USER_AGENT = "{}/{}".format(APPLICATION_NAME, app_version)
+
+
+# This is find code usage
+def get_http_client_info():
+    return http_client_info.ClientInfo(user_agent=USER_AGENT)
+
+
+bq_client = bigquery.Client(client_info=get_http_client_info())
 
 
 # This function checks if billboard dataset already exists or not
@@ -78,7 +90,7 @@ def create_dataset(args):
         # Create dataset for BB for standard export.
         dataset_id = "{}.{}".format(args.PROJECT_ID,
                                     args.BILLBOARD_DATASET_NAME_TO_BE_CREATED)
-        create_dataset_by_location(dataset_id, standard_table_info.location)
+        create_dataset_by_loc(dataset_id, standard_table_info.location)
     except NotFound:
         print("Table {} is not found check the export and proceed.".format(
             standard_source_id))
@@ -101,7 +113,7 @@ def create_dataset(args):
             dataset_id = "{}.{}".format(args.PROJECT_ID, detailedBBDataset)
             print("Creating another dataset {} in detailed export loc".format(
                 dataset_id))
-            create_dataset_by_location(dataset_id, detailed_table_info.location)
+            create_dataset_by_loc(dataset_id, detailed_table_info.location)
 
     except NotFound:
         print("Table {} is not found check the export.".format(
@@ -112,7 +124,7 @@ def create_dataset(args):
 
 # Creates billboard dataset based on billing exported location
 # Location is taken from the billing export table provided by the user.
-def create_dataset_by_location(dataset_id, location):
+def create_dataset_by_loc(dataset_id, location):
     # Check if billboard dataset exists
     if check_billboard_dataset_exists(dataset_id) is True:
         return
@@ -165,8 +177,16 @@ def create_billboard_view(args, isStandard):
         return
 
     sql = """
-    CREATE VIEW if not exists `{}`
-    AS select *, COALESCE((SELECT SUM(x.amount) FROM UNNEST(s.credits) x),0) AS credits_sum_amount, COALESCE((SELECT SUM(x.amount) FROM UNNEST(s.credits) x),0) + cost as net_cost, EXTRACT(DATE FROM _PARTITIONTIME) AS date from `{}` s WHERE _PARTITIONTIME >'2020-08-01'
+    CREATE OR REPLACE VIEW  `{}`
+    AS select *,
+    COALESCE((SELECT SUM(x.amount)
+    FROM UNNEST(s.credits) x),0) AS credits_sum_amount,
+    COALESCE((SELECT SUM(x.amount)
+    FROM UNNEST(s.credits) x),0) + cost as net_cost,
+    PARSE_DATE("%Y%m", invoice.month) AS Invoice_Month,
+    _PARTITIONDATE AS date
+    from `{}` s
+    WHERE _PARTITIONDATE > DATE_SUB(CURRENT_DATE(), INTERVAL 13 MONTH)
     """.format(view_id, source_id)
 
     # Not sure why this need project_id
@@ -179,9 +199,10 @@ def create_billboard_view(args, isStandard):
         output_url = report_base_url + standard_view_url.format(
             args.PROJECT_ID, args.BILLBOARD_DATASET_NAME_TO_BE_CREATED,
             args.bb_standard)
-    else:
-        output_url = output_url + detailed_view_url.format(
-            args.PROJECT_ID, detailedBBDataset, args.bb_detailed)
+    # not using detailed datasource yet so commenting
+    # else:
+    #     output_url = output_url + detailed_view_url.format(
+    #         args.PROJECT_ID, detailedBBDataset, args.bb_detailed)
 
     print('Created view {}{}.{}.{}'.format(Back.GREEN, job.destination.project,
                                            job.destination.dataset_id,
@@ -251,22 +272,32 @@ def main(argv):
     args = parser.parse_args()
     print('Version of billboard.py  ' + app_version + "\n")
 
-    if args.DETAILED_BILLING_EXPORT_DATASET_NAME is None:
-        print("Detailed export not provided so setting default to Standard.")
-        args.DETAILED_BILLING_EXPORT_DATASET_NAME = args.STANDARD_BILLING_EXPORT_DATASET_NAME
+    # if args.DETAILED_BILLING_EXPORT_DATASET_NAME is None:
+    # print("Detailed export not provided so setting default to Standard.")
+    # args.DETAILED_BILLING_EXPORT_DATASET_NAME
+    # = args.STANDARD_BILLING_EXPORT_DATASET_NAME
 
-    # Detailed Export could be in different region so name will be modified as {}_detail in logic
+    # Detailed Export could be in different region
+    # so name will be modified as {}_detail in logic
     # So we are storing in global variable.
     detailedBBDataset = '{}'.format(args.BILLBOARD_DATASET_NAME_TO_BE_CREATED)
 
     project_id_temp = "projects/{}".format(args.PROJECT_ID)
+
+    # Check if billing api is enabled.
+    # service = discovery.build('serviceusage', 'v1')
+    # request = service.services().get(
+    #     name=f"{project_id_temp}/services/cloudbilling.googleapis.com")
+    # response = request.execute()
+    # if response.get('state') == 'DISABLED':
+    #     print("Cloud Billing API is not enabled.")
+    #     return sys.exit(1)
+
     try:
         project_billing_info = billing.CloudBillingClient(
         ).get_project_billing_info(name=project_id_temp)
     except PermissionDenied:
-        print(
-            "Permission Denied so you do not have project level permission or provided wrong project id, please check."
-        )
+        print("Permission Denied, check project level permission.")
         return sys.exit(1)
 
     billing_account_name = project_billing_info.billing_account_name.split(


Test patch: