# -*- coding: utf-8 -*-
"""notebook.ipynb

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/10KllEMzGTWHKwkZLMq9YFnn7eLO6Y6Oy

# Latihan Membuat Machine Learning Pipeline
"""

!pip install tfx

import tensorflow as tf
from tfx.components import CsvExampleGen, StatisticsGen, SchemaGen, ExampleValidator, Transform, Trainer, Tuner
from tfx.proto import example_gen_pb2
from tfx.orchestration.experimental.interactive.interactive_context import InteractiveContext
import os

# Commented out IPython magic to ensure Python compatibility.
from google.colab import drive
drive.mount('/content/drive')
# %cd /content/drive/MyDrive/Pengumpulan Tugas/MLOps ML Pipeline

"""## Set Variable"""

PIPELINE_NAME = "sarcasm-pipeline"
SCHEMA_PIPELINE_NAME = "sarcasm-tfdv-schema"

#Directory untuk menyimpan artifact yang akan dihasilkan
PIPELINE_ROOT = os.path.join('pipelines', PIPELINE_NAME)

# Path to a SQLite DB file to use as an MLMD storage.
METADATA_PATH = os.path.join('metadata', PIPELINE_NAME, 'metadata.db')

# Output directory where created models from the pipeline will be exported.
SERVING_MODEL_DIR = os.path.join('serving_model', PIPELINE_NAME)

# from absl import logging
# logging.set_verbosity(logging.INFO)

DATA_ROOT = "data"

interactive_context = InteractiveContext(pipeline_root=PIPELINE_ROOT)

"""## Pembuatan Komponen Pengolahan Data"""

output = example_gen_pb2.Output(
    split_config = example_gen_pb2.SplitConfig(splits=[
        example_gen_pb2.SplitConfig.Split(name="train", hash_buckets=8),
        example_gen_pb2.SplitConfig.Split(name="eval", hash_buckets=2)
    ])
)
example_gen = CsvExampleGen(input_base=DATA_ROOT, output_config=output)
interactive_context.run(example_gen)

statistics_gen = StatisticsGen(
    examples=example_gen.outputs["examples"]
)
interactive_context.run(statistics_gen)

interactive_context.show(statistics_gen.outputs["statistics"])

schema_gen = SchemaGen(statistics=statistics_gen.outputs["statistics"])
interactive_context.run(schema_gen)

interactive_context.show(schema_gen.outputs["schema"])

example_validator = ExampleValidator(
    statistics=statistics_gen.outputs['statistics'],
    schema=schema_gen.outputs['schema']
)
interactive_context.run(example_validator)

interactive_context.show(example_validator.outputs['anomalies'])

TRANSFORM_MODULE_FILE = "sarcasm_transform.py"

# Commented out IPython magic to ensure Python compatibility.
# %%writefile {TRANSFORM_MODULE_FILE}
# import tensorflow as tf
# LABEL_KEY = "is_sarcastic"
# FEATURE_KEY = "headline"
# def transformed_name(key):
#     """Renaming transformed features"""
#     return key + "_xf"
# def preprocessing_fn(inputs):
#     """
#     Preprocess input features into transformed features
# 
#     Args:
#         inputs: map from feature keys to raw features.
# 
#     Return:
#         outputs: map from feature keys to transformed features.
#     """
# 
#     outputs = {}
# 
#     outputs[transformed_name(FEATURE_KEY)] = tf.strings.lower(inputs[FEATURE_KEY])
# 
#     outputs[transformed_name(LABEL_KEY)] = tf.cast(inputs[LABEL_KEY], tf.int64)
# 
#     return outputs

transform  = Transform(
    examples=example_gen.outputs['examples'],
    schema= schema_gen.outputs['schema'],
    module_file=os.path.abspath(TRANSFORM_MODULE_FILE)
)
interactive_context.run(transform)
buatlah,dokumen,rpoyek,berdasarkan,code,ini,b,indo