GitHub PullRequestReviewCommentEvent at 2014-04-08 16:58:59
Repository: falcon-regression
Actor: arpitgupta
Payload: {"comment":{"url":"https://api.github.com/repos/samarthgupta437/falcon-regression/pulls/comments/11399712","id":11399712,"diff_hunk":"@@ -0,0 +1,249 @@\n+/**\n+ * Licensed to the Apache Software Foundation (ASF) under one\n+ * or more contributor license agreements.  See the NOTICE file\n+ * distributed with this work for additional information\n+ * regarding copyright ownership.  The ASF licenses this file\n+ * to you under the Apache License, Version 2.0 (the\n+ * \"License\"); you may not use this file except in compliance\n+ * with the License.  You may obtain a copy of the License at\n+ *\n+ *     http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.falcon.regression.hcat;\n+\n+import org.apache.commons.lang.StringUtils;\n+import org.apache.falcon.regression.core.bundle.Bundle;\n+import org.apache.falcon.regression.core.enumsAndConstants.ENTITY_TYPE;\n+import org.apache.falcon.regression.core.generated.cluster.Interfacetype;\n+import org.apache.falcon.regression.core.generated.dependencies.Frequency;\n+import org.apache.falcon.regression.core.generated.feed.ActionType;\n+import org.apache.falcon.regression.core.generated.feed.ClusterType;\n+import org.apache.falcon.regression.core.helpers.ColoHelper;\n+import org.apache.falcon.regression.core.response.ServiceResponse;\n+import org.apache.falcon.regression.core.util.AssertUtil;\n+import org.apache.falcon.regression.core.util.HadoopUtil;\n+import org.apache.falcon.regression.core.util.InstanceUtil;\n+import org.apache.falcon.regression.core.util.OSUtil;\n+import org.apache.falcon.regression.core.util.Util;\n+import org.apache.falcon.regression.core.util.XmlUtil;\n+import org.apache.falcon.regression.testHelper.BaseTestClass;\n+import org.apache.hadoop.fs.FileSystem;\n+import org.apache.hive.hcatalog.api.HCatAddPartitionDesc;\n+import org.apache.hive.hcatalog.api.HCatClient;\n+import org.apache.hive.hcatalog.api.HCatCreateTableDesc;\n+import org.apache.hive.hcatalog.common.HCatException;\n+import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;\n+import org.apache.log4j.Logger;\n+import org.apache.oozie.client.CoordinatorAction;\n+import org.apache.oozie.client.OozieClient;\n+import org.joda.time.DateTime;\n+import org.joda.time.format.DateTimeFormat;\n+import org.joda.time.format.DateTimeFormatter;\n+import org.testng.Assert;\n+import org.testng.annotations.AfterMethod;\n+import org.testng.annotations.BeforeClass;\n+import org.testng.annotations.BeforeMethod;\n+import org.testng.annotations.DataProvider;\n+import org.testng.annotations.Test;\n+\n+import java.io.IOException;\n+import java.text.ParseException;\n+import java.util.ArrayList;\n+import java.util.HashMap;\n+import java.util.List;\n+import java.util.Map;\n+\n+public class HCatReplication extends BaseTestClass {\n+\n+    private static Logger logger = Logger.getLogger(HCatReplication.class);\n+    ColoHelper cluster = servers.get(0);\n+    FileSystem clusterFS = serverFS.get(0);\n+    OozieClient clusterOC = serverOC.get(0);\n+    HCatClient clusterHC = cluster.getClusterHelper().getHCatClient();\n+\n+    ColoHelper cluster2 = servers.get(1);\n+    FileSystem cluster2FS = serverFS.get(1);\n+    OozieClient cluster2OC = serverOC.get(1);\n+    HCatClient cluster2HC = cluster2.getClusterHelper().getHCatClient();\n+\n+    ColoHelper cluster3 = servers.get(2);\n+    FileSystem cluster3FS = serverFS.get(2);\n+    OozieClient cluster3OC = serverOC.get(2);\n+    HCatClient cluster3HC = cluster3.getClusterHelper().getHCatClient();\n+\n+    final String baseTestHDFSDir = baseHDFSDir + \"/HCatReplication\";\n+\n+    final String dbName = \"default\";\n+    private static final String localHCatData = OSUtil.getPath(OSUtil.RESOURCES, \"hcat\", \"data\");\n+\n+    @BeforeClass\n+    public void beforeClass() throws IOException {\n+        // create the base dir on all clusters.\n+        // method will delete the dir if it exists.\n+        HadoopUtil.createDir(baseTestHDFSDir, clusterFS, cluster2FS, cluster3FS);\n+    }\n+\n+    @BeforeMethod\n+    public void setUp() throws Exception {\n+        Bundle bundle = Util.readHCatBundle();\n+        bundles[0] = new Bundle(bundle, cluster.getEnvFileName(), cluster.getPrefix());\n+        bundles[0].generateUniqueBundle();\n+        bundles[0].setClusterInterface(Interfacetype.REGISTRY, cluster.getClusterHelper().getHCatEndpoint());\n+\n+        bundles[1] = new Bundle(bundle, cluster2.getEnvFileName(), cluster2.getPrefix());\n+        bundles[1].generateUniqueBundle();\n+        bundles[1].setClusterInterface(Interfacetype.REGISTRY, cluster2.getClusterHelper()\n+                .getHCatEndpoint());\n+\n+        bundles[2] = new Bundle(bundle, cluster3.getEnvFileName(), cluster3.getPrefix());\n+        bundles[2].generateUniqueBundle();\n+        bundles[2].setClusterInterface(Interfacetype.REGISTRY, cluster3.getClusterHelper()\n+                .getHCatEndpoint());\n+\n+    }\n+\n+    @DataProvider\n+    public String[][] generateSeparators() {\n+        //disabling till FALCON-372 is fixed\n+        //return new String[][] {{\"-\"}, {\"/\"}};\n+        return new String[][] {{\"-\"}, };\n+    }\n+\n+    // make sure oozie changes mentioned FALCON-389 are done on the clusters. Otherwise the test\n+    // will fail.\n+    // HIVE-6848 also needs to be available in hive for the test to work.\n+    @Test(dataProvider = \"generateSeparators\")\n+    public void oneSourceOneTarget(String separator) throws Exception {\n+        String tcName = \"HCatReplication_oneSourceOneTarget\";\n+        if (separator.equals(\"-\")) {\n+            tcName += \"_hyphen\";\n+        } else {\n+            tcName += \"_slash\";\n+        }\n+        String tblName = tcName;\n+        String testHdfsDir = baseTestHDFSDir + \"/\" + tcName;\n+        HadoopUtil.createDir(testHdfsDir, clusterFS, cluster2FS);\n+        final String tableUriPartitionFragment = StringUtils.join(new String[] {\"#dt=${YEAR}\", \"${MONTH}\", \"${DAY}\", \"${HOUR}\"}, separator);\n+        String tableUri = \"catalog:\" + dbName + \":\" + tblName + tableUriPartitionFragment;\n+        final String startDate = \"2010-01-01T20:00Z\";\n+        final String endDate = \"2010-01-02T04:00Z\";\n+        final String datePattern = StringUtils.join(new String[]{\"yyyy\", \"MM\", \"dd\", \"HH\"}, separator);\n+        List<String> dataDates = getDatesList(startDate, endDate, datePattern, 60);\n+\n+        final ArrayList<String> dataset = createPeriodicDataset(dataDates, localHCatData, clusterFS, testHdfsDir);\n+        final String col1Name = \"id\";\n+        final String col2Name = \"value\";\n+        final String partitionColumn = \"dt\";\n+\n+        ArrayList<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>();\n+        cols.add(new HCatFieldSchema(col1Name, HCatFieldSchema.Type.STRING, col1Name + \" comment\"));\n+        cols.add(new HCatFieldSchema(col2Name, HCatFieldSchema.Type.STRING, col2Name + \" comment\"));\n+        ArrayList<HCatFieldSchema> partitionCols = new ArrayList<HCatFieldSchema>();\n+\n+        clusterHC.dropTable(dbName, tblName, true);\n+        partitionCols.add(new HCatFieldSchema(partitionColumn, HCatFieldSchema.Type.STRING, partitionColumn + \" partition\"));\n+        // create the table on cluster 1\n+        clusterHC.createTable(HCatCreateTableDesc\n+                .create(dbName, tblName, cols)\n+                .partCols(partitionCols)\n+                .ifNotExists(true)\n+                .isTableExternal(true)\n+                .location(testHdfsDir)\n+                .build());\n+        addPartitionsToTable(dataDates, dataset, \"dt\", dbName, tblName, clusterHC);\n+        // create table on cluster 2 if we are copying data from cluster 1 to cluster2 then the\n+        // table also needs to exist on cluster 2\n+        cluster2HC.dropTable(dbName, tblName, true);\n+        cluster2HC.createTable(HCatCreateTableDesc\n+                .create(dbName, tblName, cols)\n+                .partCols(partitionCols)\n+                .ifNotExists(true)\n+                .isTableExternal(true)\n+                .location(testHdfsDir)\n+                .build());\n+\n+        Bundle.submitCluster(bundles[0], bundles[1]);\n+\n+        bundles[0].setInputFeedPeriodicity(1, Frequency.TimeUnit.hours);\n+        bundles[0].setInputFeedValidity(startDate, \"2099-01-01T00:00Z\");\n+        bundles[0].setInputFeedTableUri(tableUri);\n+\n+        String feed = bundles[0].getDataSets().get(0);\n+        // set the cluster 2 as the target.\n+        feed = InstanceUtil.setFeedClusterWithTable(feed,\n+                XmlUtil.createValidity(startDate, endDate),\n+                XmlUtil.createRtention(\"months(9000)\", ActionType.DELETE),\n+                Util.readClusterName(bundles[1].getClusters().get(0)), ClusterType.TARGET, null,\n+                tableUri, null);\n+\n+        AssertUtil.assertSucceeded(\n+                prism.getFeedHelper().submitAndSchedule(Util.URLS.SUBMIT_AND_SCHEDULE_URL,\n+                        feed));\n+        Thread.sleep(15000);\n+        //check if all coordinators exist\n+        Assert.assertEquals(InstanceUtil\n+                .checkIfFeedCoordExist(cluster2.getFeedHelper(), Util.readDatasetName(feed),\n+                        \"REPLICATION\"), 1);\n+\n+        //replication should start, wait while it ends\n+        InstanceUtil.waitTillInstanceReachState(cluster2OC, Util.readEntityName(feed), 1,\n+                CoordinatorAction.Status.SUCCEEDED, 5, ENTITY_TYPE.FEED);\n+\n+\n+        //TODO: Add mode checks. Currently job fails because of HIVE-6848\n+    }\n+\n+    private void addPartitionsToTable(List<String> partitions, List<String> partitionLocations, String partitionCol,","path":"merlin/src/test/java/org/apache/falcon/regression/hcat/HCatReplication.java","position":206,"original_position":203,"commit_id":"e46f5a8219a870a267927cde6bc5e99d3963d017","original_commit_id":"1bb0699ce9ed9daf7f47a650001e9444cc3dad17","user":{"login":"arpitgupta","id":901701,"avatar_url":"https://avatars.githubusercontent.com/u/901701?","gravatar_id":"8da6f56d6cdb3a4561f2d2e4acbdfdf7","url":"https://api.github.com/users/arpitgupta","html_url":"https://github.com/arpitgupta","followers_url":"https://api.github.com/users/arpitgupta/followers","following_url":"https://api.github.com/users/arpitgupta/following{/other_user}","gists_url":"https://api.github.com/users/arpitgupta/gists{/gist_id}","starred_url":"https://api.github.com/users/arpitgupta/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/arpitgupta/subscriptions","organizations_url":"https://api.github.com/users/arpitgupta/orgs","repos_url":"https://api.github.com/users/arpitgupta/repos","events_url":"https://api.github.com/users/arpitgupta/events{/privacy}","received_events_url":"https://api.github.com/users/arpitgupta/received_events","type":"User","site_admin":false},"body":"Can we do that post merge? I am going to add more tests and clean up of these method from both the classes i want to do last once i have a base line.","created_at":"2014-04-08T16:58:59Z","updated_at":"2014-04-08T16:58:59Z","html_url":"https://github.com/samarthgupta437/falcon-regression/pull/169#discussion_r11399712","pull_request_url":"https://api.github.com/repos/samarthgupta437/falcon-regression/pulls/169","_links":{"self":{"href":"https://api.github.com/repos/samarthgupta437/falcon-regression/pulls/comments/11399712"},"html":{"href":"https://github.com/samarthgupta437/falcon-regression/pull/169#discussion_r11399712"},"pull_request":{"href":"https://api.github.com/repos/samarthgupta437/falcon-regression/pulls/169"}}}}