Repository: sql-machine-learning/elasticdl

Issue:
Rename arguments "--*_data_dir" to "--*_data"
Currently, if we want to use ODPS data source, we can pass ODPS table names to "--*_data_dir" arguments instead of previously passing file directories if we want to use RecordIO files as data source. I am proposing the following changes so the data sources are not "file"-dependent:
```
training_data_dir -> training_data
evaluation_data_dir -> evaluation_data
prediction_data_dir -> prediction_data
```
We can add help information like this:
```
This can be either the data directory that contains RecordIO files or an ODPS table name used for training.
```
If there are any suggestions on other names, please let me know in this issue as well.


Discussion:
Is it better to append a string "path" at the end?
@chunyang-wen But in the ODPS case, this should be a table name instead. Either a "path" or "directory" doesn't make much sense.
@terrytangyuan For ODPS, path=odps://project.table. Previously we use path for ODPS table. I agree with you.

Resolving patch:
diff --git a/elasticdl/python/common/args.py b/elasticdl/python/common/args.py
--- a/elasticdl/python/common/args.py
+++ b/elasticdl/python/common/args.py
@@ -15,17 +15,17 @@
     "grads_to_wait",
     "num_epochs",
     "tensorboard_log_dir",
-    "training_data_dir",
+    "training_data",
 ]
 
 EVALUATION_GROUP = [
     "evaluation_steps",
-    "evaluation_data_dir",
+    "evaluation_data",
     "evaluation_start_delay_secs",
     "evaluation_throttle_secs",
 ]
 
-PREDICTION_GROUP = ["prediction_data_dir"]
+PREDICTION_GROUP = ["prediction_data"]
 
 CHECKPOINT_GROUP = [
     "checkpoint_filename_for_init",
@@ -218,13 +218,15 @@ def add_train_params(parser):
         default=1,
     )
     parser.add_argument(
-        "--training_data_dir",
-        help="Training data directory. Files should be in RecordIO format",
+        "--training_data",
+        help="Either the data directory that contains RecordIO files "
+        "or an ODPS table name used for training.",
         default="",
     )
     parser.add_argument(
-        "--evaluation_data_dir",
-        help="Evaluation data directory. Files should be in RecordIO format",
+        "--evaluation_data",
+        help="Either the data directory that contains RecordIO files "
+        "or an ODPS table name used for evaluation.",
         default="",
     )
     parser.add_argument(
@@ -297,8 +299,9 @@ def add_train_params(parser):
 
 def add_evaluate_params(parser):
     parser.add_argument(
-        "--evaluation_data_dir",
-        help="Evaluation data directory. Files should be in RecordIO format",
+        "--evaluation_data",
+        help="Either the data directory that contains RecordIO files "
+        "or an ODPS table name used for evaluation.",
         required=True,
     )
     parser.add_argument(
@@ -310,8 +313,9 @@ def add_evaluate_params(parser):
 
 def add_predict_params(parser):
     parser.add_argument(
-        "--prediction_data_dir",
-        help="Prediction data directory. Files should be in RecordIO format",
+        "--prediction_data",
+        help="Either the data directory that contains RecordIO files "
+        "or an ODPS table name used for prediction.",
         required=True,
     )
     parser.add_argument(
@@ -463,8 +467,9 @@ def parse_master_args(master_args=None):
         "--worker_pod_priority", help="Priority requested by workers"
     )
     parser.add_argument(
-        "--prediction_data_dir",
-        help="Prediction data directory. Files should be in RecordIO format",
+        "--prediction_data",
+        help="Either the data directory that contains RecordIO files "
+        "or an ODPS table name used for prediction.",
         default="",
     )
     add_common_params(parser)
@@ -478,23 +483,21 @@ def parse_master_args(master_args=None):
     if all(
         v == "" or v is None
         for v in [
-            args.training_data_dir,
-            args.evaluation_data_dir,
-            args.prediction_data_dir,
+            args.training_data,
+            args.evaluation_data,
+            args.prediction_data,
         ]
     ):
         raise ValueError(
             "At least one of the data directories needs to be provided"
         )
 
-    if args.prediction_data_dir and (
-        args.training_data_dir or args.evaluation_data_dir
-    ):
+    if args.prediction_data and (args.training_data or args.evaluation_data):
         raise ValueError(
             "Running prediction together with training or evaluation "
             "is not supported"
         )
-    if args.prediction_data_dir and not args.checkpoint_filename_for_init:
+    if args.prediction_data and not args.checkpoint_filename_for_init:
         raise ValueError(
             "checkpoint_filename_for_init is required for running "
             "prediction job"
diff --git a/elasticdl/python/master/main.py b/elasticdl/python/master/main.py
--- a/elasticdl/python/master/main.py
+++ b/elasticdl/python/master/main.py
@@ -36,9 +36,9 @@
 
 
 def _make_task_dispatcher(
-    training_data_dir,
-    evaluation_data_dir,
-    prediction_data_dir,
+    training_data,
+    evaluation_data,
+    prediction_data,
     records_per_task,
     num_epochs,
 ):
@@ -53,11 +53,11 @@ def _maybe_create_shards(data_origin):
             else {}
         )
 
-    prediction_f_records = _maybe_create_shards(prediction_data_dir)
+    prediction_f_records = _maybe_create_shards(prediction_data)
 
     return _TaskDispatcher(
-        _maybe_create_shards(training_data_dir),
-        _maybe_create_shards(evaluation_data_dir),
+        _maybe_create_shards(training_data),
+        _maybe_create_shards(evaluation_data),
         prediction_f_records,
         records_per_task,
         # Only generate prediction tasks for 1 epoch
@@ -90,16 +90,16 @@ def main():
         "Starting task queue with training data directory %s, "
         "evaluation data directory %s, "
         "and prediction data directory %s",
-        args.training_data_dir,
-        args.evaluation_data_dir,
-        args.prediction_data_dir,
+        args.training_data,
+        args.evaluation_data,
+        args.prediction_data,
     )
 
     records_per_task = args.minibatch_size * args.num_minibatches_per_task
     task_d = _make_task_dispatcher(
-        args.training_data_dir,
-        args.evaluation_data_dir,
-        args.prediction_data_dir,
+        args.training_data,
+        args.evaluation_data,
+        args.prediction_data,
         records_per_task,
         args.num_epochs,
     )
@@ -113,25 +113,25 @@ def main():
 
     if all(
         (
-            args.training_data_dir,
-            args.evaluation_data_dir,
+            args.training_data,
+            args.evaluation_data,
             args.evaluation_throttle_secs or args.evaluation_steps,
         )
     ):
         job_type = JobType.TRAINING_WITH_EVALUATION
     elif all(
         (
-            args.evaluation_data_dir,
-            not args.training_data_dir,
-            not args.prediction_data_dir,
+            args.evaluation_data,
+            not args.training_data,
+            not args.prediction_data,
         )
     ):
         job_type = JobType.EVALUATION_ONLY
     elif all(
         (
-            args.prediction_data_dir,
-            not args.evaluation_data_dir,
-            not args.training_data_dir,
+            args.prediction_data,
+            not args.evaluation_data,
+            not args.training_data,
         )
     ):
         job_type = JobType.PREDICTION_ONLY


Test patch:
diff --git a/scripts/client_test.sh b/scripts/client_test.sh
--- a/scripts/client_test.sh
+++ b/scripts/client_test.sh
@@ -7,8 +7,8 @@ if [[ "$JOB_TYPE" == "train" ]]; then
       --image_base=elasticdl:ci \
       --model_zoo=model_zoo \
       --model_def=mnist_functional_api.mnist_functional_api.custom_model \
-      --training_data_dir=/data/mnist/train \
-      --evaluation_data_dir=/data/mnist/test \
+      --training_data=/data/mnist/train \
+      --evaluation_data=/data/mnist/test \
       --num_epochs=2 \
       --master_resource_request="cpu=400m,memory=1024Mi" \
       --master_resource_limit="cpu=1,memory=2048Mi" \
@@ -30,7 +30,7 @@ elif [[ "$JOB_TYPE" == "evaluate" ]]; then
       --model_zoo=model_zoo \
       --model_def=mnist_functional_api.mnist_functional_api.custom_model \
       --checkpoint_filename_for_init=elasticdl/python/tests/testdata/mnist_functional_api_model_v110.chkpt \
-      --evaluation_data_dir=/data/mnist/test \
+      --evaluation_data=/data/mnist/test \
       --num_epochs=1 \
       --master_resource_request="cpu=400m,memory=1024Mi" \
       --master_resource_limit="cpu=1,memory=2048Mi" \
@@ -49,7 +49,7 @@ elif [[ "$JOB_TYPE" == "predict" ]]; then
       --model_zoo=model_zoo \
       --model_def=mnist_functional_api.mnist_functional_api.custom_model \
       --checkpoint_filename_for_init=elasticdl/python/tests/testdata/mnist_functional_api_model_v110.chkpt \
-      --prediction_data_dir=/data/mnist/test \
+      --prediction_data=/data/mnist/test \
       --master_resource_request="cpu=400m,memory=1024Mi" \
       --master_resource_limit="cpu=1,memory=2048Mi" \
       --worker_resource_request="cpu=400m,memory=2048Mi" \
@@ -65,7 +65,7 @@ elif [[ "$JOB_TYPE" == "odps" ]]; then
       --image_base=elasticdl:ci \
       --model_zoo=model_zoo \
       --model_def=odps_iris_dnn_model.odps_iris_dnn_model.custom_model \
-      --training_data_dir=$ODPS_TABLE_NAME \
+      --training_data=$ODPS_TABLE_NAME \
       --data_reader_params='columns=["sepal_length", "sepal_width", "petal_length", "petal_width", "class"]' \
       --envs="ODPS_PROJECT_NAME=$ODPS_PROJECT_NAME,ODPS_ACCESS_ID=$ODPS_ACCESS_ID,ODPS_ACCESS_KEY=$ODPS_ACCESS_KEY" \
       --num_epochs=2 \