Spaces:
Sleeping
Sleeping
hakim
commited on
Commit
•
a637525
1
Parent(s):
e8bb63c
module updaed
Browse files- src/textsummarizer/config/configuration.py +3 -3
- src/textsummarizer/conponents/data_ingestion.py +3 -3
- src/textsummarizer/conponents/data_tranformation.py +1 -1
- src/textsummarizer/conponents/data_validation.py +2 -2
- src/textsummarizer/conponents/model_evaluation.py +1 -1
- src/textsummarizer/conponents/model_trainer.py +2 -2
- src/textsummarizer/pipeline/stage_01_data_ingestion.py +3 -3
- src/textsummarizer/pipeline/stage_02_data_validation.py +3 -3
- src/textsummarizer/pipeline/stage_03_data_transformation.py +2 -2
- src/textsummarizer/pipeline/stage_04_model_trainer.py +2 -2
- src/textsummarizer/pipeline/stage_05_model_evaluation.py +2 -2
- src/textsummarizer/utils/common.py +1 -1
src/textsummarizer/config/configuration.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1 |
-
from textsummarizer.constants import *
|
2 |
-
from textsummarizer.utils.common import read_yaml, create_directories
|
3 |
-
from textsummarizer.entity.config_entity import (DataIngestionConfig,
|
4 |
DataValidationConfig,
|
5 |
DataTransformationConfig,
|
6 |
ModelTrainerConfig,
|
|
|
1 |
+
from src.textsummarizer.constants import *
|
2 |
+
from src.textsummarizer.utils.common import read_yaml, create_directories
|
3 |
+
from src.textsummarizer.entity.config_entity import (DataIngestionConfig,
|
4 |
DataValidationConfig,
|
5 |
DataTransformationConfig,
|
6 |
ModelTrainerConfig,
|
src/textsummarizer/conponents/data_ingestion.py
CHANGED
@@ -1,9 +1,9 @@
|
|
1 |
import os
|
2 |
import urllib.request as request
|
3 |
import zipfile
|
4 |
-
from textsummarizer.logging import logger
|
5 |
-
from textsummarizer.utils.common import get_size
|
6 |
-
from textsummarizer.entity.config_entity import DataIngestionConfig
|
7 |
from pathlib import Path
|
8 |
|
9 |
|
|
|
1 |
import os
|
2 |
import urllib.request as request
|
3 |
import zipfile
|
4 |
+
from src.textsummarizer.logging import logger
|
5 |
+
from src.textsummarizer.utils.common import get_size
|
6 |
+
from src.textsummarizer.entity.config_entity import DataIngestionConfig
|
7 |
from pathlib import Path
|
8 |
|
9 |
|
src/textsummarizer/conponents/data_tranformation.py
CHANGED
@@ -1,5 +1,5 @@
|
|
1 |
import os
|
2 |
-
from textsummarizer.logging import logger
|
3 |
from transformers import AutoTokenizer
|
4 |
from datasets import load_dataset, load_from_disk
|
5 |
from textsummarizer.entity.config_entity import DataTransformationConfig
|
|
|
1 |
import os
|
2 |
+
from src.textsummarizer.logging import logger
|
3 |
from transformers import AutoTokenizer
|
4 |
from datasets import load_dataset, load_from_disk
|
5 |
from textsummarizer.entity.config_entity import DataTransformationConfig
|
src/textsummarizer/conponents/data_validation.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1 |
import os
|
2 |
-
from textsummarizer.logging import logger
|
3 |
-
from textsummarizer.entity.config_entity import DataValidationConfig
|
4 |
|
5 |
class DataValiadtion:
|
6 |
def __init__(self, config: DataValidationConfig):
|
|
|
1 |
import os
|
2 |
+
from src.textsummarizer.logging import logger
|
3 |
+
from src.textsummarizer.entity.config_entity import DataValidationConfig
|
4 |
|
5 |
class DataValiadtion:
|
6 |
def __init__(self, config: DataValidationConfig):
|
src/textsummarizer/conponents/model_evaluation.py
CHANGED
@@ -3,7 +3,7 @@ from datasets import load_dataset, load_from_disk, load_metric
|
|
3 |
import torch
|
4 |
import pandas as pd
|
5 |
from tqdm import tqdm
|
6 |
-
from textsummarizer.entity.config_entity import ModelEvaluationConfig
|
7 |
import mlflow
|
8 |
import dagshub
|
9 |
import json
|
|
|
3 |
import torch
|
4 |
import pandas as pd
|
5 |
from tqdm import tqdm
|
6 |
+
from src.textsummarizer.entity.config_entity import ModelEvaluationConfig
|
7 |
import mlflow
|
8 |
import dagshub
|
9 |
import json
|
src/textsummarizer/conponents/model_trainer.py
CHANGED
@@ -2,7 +2,7 @@ from transformers import TrainingArguments, Trainer
|
|
2 |
from transformers import DataCollatorForSeq2Seq
|
3 |
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
|
4 |
from datasets import load_dataset, load_from_disk
|
5 |
-
from textsummarizer.entity.config_entity import ModelTrainerConfig
|
6 |
import torch
|
7 |
import os
|
8 |
|
@@ -38,7 +38,7 @@ class ModelTrainer:
|
|
38 |
tokenizer=tokenizer, data_collator=seq2seq_data_collator,
|
39 |
train_dataset=dataset_samsum_pt["train"],
|
40 |
eval_dataset=dataset_samsum_pt["validation"])
|
41 |
-
|
42 |
|
43 |
trainer.train()
|
44 |
|
|
|
2 |
from transformers import DataCollatorForSeq2Seq
|
3 |
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
|
4 |
from datasets import load_dataset, load_from_disk
|
5 |
+
from src.textsummarizer.entity.config_entity import ModelTrainerConfig
|
6 |
import torch
|
7 |
import os
|
8 |
|
|
|
38 |
tokenizer=tokenizer, data_collator=seq2seq_data_collator,
|
39 |
train_dataset=dataset_samsum_pt["train"],
|
40 |
eval_dataset=dataset_samsum_pt["validation"])
|
41 |
+
|
42 |
|
43 |
trainer.train()
|
44 |
|
src/textsummarizer/pipeline/stage_01_data_ingestion.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1 |
-
from textsummarizer.config.configuration import ConfigurationManager
|
2 |
-
from textsummarizer.entity.config_entity import DataIngestionConfig
|
3 |
-
from textsummarizer.conponents.data_ingestion import DataIngestion
|
4 |
|
5 |
|
6 |
class DataIngestionPipeline:
|
|
|
1 |
+
from src.textsummarizer.config.configuration import ConfigurationManager
|
2 |
+
from src.textsummarizer.entity.config_entity import DataIngestionConfig
|
3 |
+
from src.textsummarizer.conponents.data_ingestion import DataIngestion
|
4 |
|
5 |
|
6 |
class DataIngestionPipeline:
|
src/textsummarizer/pipeline/stage_02_data_validation.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1 |
-
from textsummarizer.config.configuration import ConfigurationManager
|
2 |
-
from textsummarizer.entity.config_entity import DataValidationConfig
|
3 |
-
from textsummarizer.conponents.data_validation import DataValiadtion
|
4 |
|
5 |
|
6 |
class DataValidationPipeline:
|
|
|
1 |
+
from src.textsummarizer.config.configuration import ConfigurationManager
|
2 |
+
from src.textsummarizer.entity.config_entity import DataValidationConfig
|
3 |
+
from src.textsummarizer.conponents.data_validation import DataValiadtion
|
4 |
|
5 |
|
6 |
class DataValidationPipeline:
|
src/textsummarizer/pipeline/stage_03_data_transformation.py
CHANGED
@@ -1,5 +1,5 @@
|
|
1 |
-
from textsummarizer.conponents.data_tranformation import DataTransformation
|
2 |
-
from textsummarizer.config.configuration import ConfigurationManager
|
3 |
|
4 |
class DataTransformationPipeline:
|
5 |
def __init__(self):
|
|
|
1 |
+
from src.textsummarizer.conponents.data_tranformation import DataTransformation
|
2 |
+
from src.textsummarizer.config.configuration import ConfigurationManager
|
3 |
|
4 |
class DataTransformationPipeline:
|
5 |
def __init__(self):
|
src/textsummarizer/pipeline/stage_04_model_trainer.py
CHANGED
@@ -1,5 +1,5 @@
|
|
1 |
-
from textsummarizer.conponents.model_trainer import ModelTrainer
|
2 |
-
from textsummarizer.config.configuration import ConfigurationManager
|
3 |
|
4 |
class ModelTrainerPipeline:
|
5 |
def __init__(self):
|
|
|
1 |
+
from src.textsummarizer.conponents.model_trainer import ModelTrainer
|
2 |
+
from src.textsummarizer.config.configuration import ConfigurationManager
|
3 |
|
4 |
class ModelTrainerPipeline:
|
5 |
def __init__(self):
|
src/textsummarizer/pipeline/stage_05_model_evaluation.py
CHANGED
@@ -1,5 +1,5 @@
|
|
1 |
-
from textsummarizer.conponents.model_evaluation import ModelEvaluation
|
2 |
-
from textsummarizer.config.configuration import ConfigurationManager
|
3 |
|
4 |
|
5 |
class ModelEvaluationPipeline:
|
|
|
1 |
+
from src.textsummarizer.conponents.model_evaluation import ModelEvaluation
|
2 |
+
from src.textsummarizer.config.configuration import ConfigurationManager
|
3 |
|
4 |
|
5 |
class ModelEvaluationPipeline:
|
src/textsummarizer/utils/common.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1 |
import os
|
2 |
from box.exceptions import BoxValueError
|
3 |
import yaml
|
4 |
-
from textsummarizer.logging import logger
|
5 |
from ensure import ensure_annotations
|
6 |
from box import ConfigBox
|
7 |
from pathlib import Path
|
|
|
1 |
import os
|
2 |
from box.exceptions import BoxValueError
|
3 |
import yaml
|
4 |
+
from src.textsummarizer.logging import logger
|
5 |
from ensure import ensure_annotations
|
6 |
from box import ConfigBox
|
7 |
from pathlib import Path
|