Add tests and split ZeppelinRestApiTest into two files

spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java

@@ -33,7 +33,6 @@ import java.util.List;
 
 import org.apache.spark.SparkContext;
 import org.apache.spark.sql.SQLContext;
-import org.apache.spark.sql.SQLContext.QueryExecution;
 import org.apache.spark.sql.catalyst.expressions.Attribute;
 import org.apache.spark.sql.hive.HiveContext;
 import org.apache.zeppelin.display.AngularObject;

spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java

@@ -48,7 +48,6 @@ import scala.Console;
 /**
  *
  */
-@Deprecated
 public class SparkDependencyContext {
   List<Dependency> dependencies = new LinkedList<Dependency>();
   List<Repository> repositories = new LinkedList<Repository>();

zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java

@@ -17,14 +17,18 @@
 
 package org.apache.zeppelin.dep;
 
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
 
 import java.io.File;
+import java.util.Collections;
 
 import org.apache.commons.io.FileUtils;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.sonatype.aether.RepositoryException;
 
 public class DependencyResolverTest {
   private static DependencyResolver resolver;
@@ -50,13 +54,47 @@ public class DependencyResolverTest {
   public static void tearDown() throws Exception {
     FileUtils.deleteDirectory(new File(home + "/" + testPath));
     FileUtils.deleteDirectory(new File(home + "/" + testCopyPath)); 
-  }    
-  
+  }
+
+  @Rule
+  public final ExpectedException exception = ExpectedException.none();
+
+  @Test
+  public void testAddRepo() {
+    int reposCnt = resolver.getRepos().size();
+    resolver.addRepo("securecentral", "https://repo1.maven.org/maven2", false);
+    assertEquals(reposCnt + 1, resolver.getRepos().size());
+  }
+
+  @Test
+  public void testDelRepo() {
+    int reposCnt = resolver.getRepos().size();
+    resolver.delRepo("securecentral");
+    resolver.delRepo("badId");
+    assertEquals(reposCnt - 1, resolver.getRepos().size());
+  }
+
   @Test
   public void testLoad() throws Exception {
-    resolver.load("org.apache.commons:commons-lang3:3.4", testCopyPath);
+    // basic load
+    resolver.load("com.databricks:spark-csv_2.10:1.3.0", testCopyPath);
+    assertEquals(new File(home + "/" + testCopyPath).list().length, 4);
+    FileUtils.cleanDirectory(new File(home + "/" + testCopyPath));
 
-    assertTrue(new File(home + "/" + testPath + "/org/apache/commons/commons-lang3/3.4/").exists());
-    assertTrue(new File(home + "/" + testCopyPath + "/commons-lang3-3.4.jar").exists());
+    // load with exclusions parameter
+    resolver.load("com.databricks:spark-csv_2.10:1.3.0",
+        Collections.singletonList("org.scala-lang:scala-library"), testCopyPath);
+    assertEquals(new File(home + "/" + testCopyPath).list().length, 3);
+    FileUtils.cleanDirectory(new File(home + "/" + testCopyPath));
+
+    // load from added repository
+    resolver.addRepo("sonatype", "https://oss.sonatype.org/content/repositories/agimatec-releases/", false);
+    resolver.load("com.agimatec:agimatec-validation:0.9.3", testCopyPath);
+    assertEquals(new File(home + "/" + testCopyPath).list().length, 8);
+
+    // load invalid artifact
+    resolver.delRepo("sonatype");
+    exception.expect(RepositoryException.class);
+    resolver.load("com.agimatec:agimatec-validation:0.9.3", testCopyPath);
   }
 }

zeppelin-server/src/test/java/org/apache/zeppelin/ZeppelinIT.java

@@ -17,20 +17,17 @@
 
 package org.apache.zeppelin;
 
-import com.google.common.base.Function;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.openqa.selenium.*;
-import org.openqa.selenium.support.ui.FluentWait;
-import org.openqa.selenium.support.ui.Wait;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
-import java.util.concurrent.TimeUnit;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 /**
  * Test Zeppelin with web browser.
@@ -202,4 +199,49 @@ public class ZeppelinIT extends AbstractZeppelinIT {
 
     }
   }
+
+  @Test
+  public void testSparkInterpreterDependencyLoading() {
+    // navigate to interpreter page
+    WebElement interpreterLink = driver.findElement(By.linkText("Interpreter"));
+    interpreterLink.click();
+
+    // add new dependency to spark interpreter
+    WebElement sparkEditBtn = pollingWait(By.xpath("//div[h3[text()[contains(.,'spark')]]]//button[contains(.,'edit')]"),
+        MAX_BROWSER_TIMEOUT_SEC);
+    sparkEditBtn.click();
+    WebElement depArtifact = driver.findElement(By.xpath("//input[@ng-model='setting.depArtifact']"));
+    String artifact = "org.apache.commons:commons-csv:1.1";
+    depArtifact.sendKeys(artifact);
+    driver.findElement(By.xpath("//button[contains(.,'Save')]")).submit();
+    driver.switchTo().alert().accept();
+
+    driver.navigate().back();
+    createNewNote();
+
+    // wait for first paragraph's " READY " status text
+    waitForParagraph(1, "READY");
+
+    WebElement paragraph1Editor = driver.findElement(By.xpath(getParagraphXPath(1) + "//textarea"));
+
+    paragraph1Editor.sendKeys("import org.apache.commons.csv.CSVFormat");
+    paragraph1Editor.sendKeys(Keys.chord(Keys.SHIFT, Keys.ENTER));
+    waitForParagraph(1, "FINISHED");
+
+    // check expected text
+    assertTrue(waitForText("import org.apache.commons.csv.CSVFormat",
+        By.xpath(getParagraphXPath(1) + "//div[starts-with(@id, 'p') and contains(@id, 'text')]/div")));
+
+    // reset dependency
+    interpreterLink.click();
+    sparkEditBtn = pollingWait(By.xpath("//div[h3[text()[contains(.,'spark')]]]//button[contains(.,'edit')]"),
+        MAX_BROWSER_TIMEOUT_SEC);
+    sparkEditBtn.click();
+    WebElement testDepRemoveBtn = driver.findElement(By.xpath("//tr[descendant::text()[contains(.,'" +
+        artifact + "')]]/td[3]/div"));
+    sleep(5000, true);
+    testDepRemoveBtn.click();
+    driver.findElement(By.xpath("//button[contains(.,'Save')]")).submit();
+    driver.switchTo().alert().accept();
+  }
 }

zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java

@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.rest;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.httpclient.methods.DeleteMethod;
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.commons.httpclient.methods.PostMethod;
+import org.apache.commons.httpclient.methods.PutMethod;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.zeppelin.interpreter.InterpreterSetting;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.notebook.Paragraph;
+import org.apache.zeppelin.scheduler.Job.Status;
+import org.apache.zeppelin.server.ZeppelinServer;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.FixMethodOrder;
+import org.junit.Test;
+import org.junit.runners.MethodSorters;
+
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
+
+import static org.junit.Assert.*;
+
+/**
+ * Zeppelin interpreter rest api tests
+ */
+@FixMethodOrder(MethodSorters.NAME_ASCENDING)
+public class InterpreterRestApiTest extends AbstractTestRestApi {
+  Gson gson = new Gson();
+
+  @BeforeClass
+  public static void init() throws Exception {
+    AbstractTestRestApi.startUp();
+  }
+
+  @AfterClass
+  public static void destroy() throws Exception {
+    AbstractTestRestApi.shutDown();
+  }
+
+  @Test
+  public void getAvailableInterpreters() throws IOException {
+    // when
+    GetMethod get = httpGet("/interpreter");
+
+    // then
+    assertThat(get, isAllowed());
+    Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
+    }.getType());
+    Map<String, Object> body = (Map<String, Object>) resp.get("body");
+    assertEquals(ZeppelinServer.notebook.getInterpreterFactory().getRegisteredInterpreterList().size(), body.size());
+    get.releaseConnection();
+  }
+
+  @Test
+  public void getSettings() throws IOException {
+    // when
+    GetMethod get = httpGet("/interpreter/setting");
+
+    // then
+    Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
+    }.getType());
+    assertThat(get, isAllowed());
+    get.releaseConnection();
+  }
+
+  @Test
+  public void testSettingsCRUD() throws IOException {
+    // Call Create Setting REST API
+    String jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"propvalue\"}," +
+        "\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
+        "\"dependencies\":[]}";
+    PostMethod post = httpPost("/interpreter/setting/", jsonRequest);
+    LOG.info("testSettingCRUD create response\n" + post.getResponseBodyAsString());
+    assertThat("test create method:", post, isCreated());
+
+    Map<String, Object> resp = gson.fromJson(post.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
+    }.getType());
+    Map<String, Object> body = (Map<String, Object>) resp.get("body");
+    //extract id from body string {id=2AWMQDNX7, name=md2, group=md,
+    String newSettingId = body.toString().split(",")[0].split("=")[1];
+    post.releaseConnection();
+
+    // Call Update Setting REST API
+    jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"Otherpropvalue\"}," +
+        "\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
+        "\"dependencies\":[]}";
+    PutMethod put = httpPut("/interpreter/setting/" + newSettingId, jsonRequest);
+    LOG.info("testSettingCRUD update response\n" + put.getResponseBodyAsString());
+    assertThat("test update method:", put, isAllowed());
+    put.releaseConnection();
+
+    // Call Delete Setting REST API
+    DeleteMethod delete = httpDelete("/interpreter/setting/" + newSettingId);
+    LOG.info("testSettingCRUD delete response\n" + delete.getResponseBodyAsString());
+    assertThat("Test delete method:", delete, isAllowed());
+    delete.releaseConnection();
+  }
+
+  @Test
+  public void testInterpreterAutoBinding() throws IOException {
+    // create note
+    Note note = ZeppelinServer.notebook.createNote();
+
+    // check interpreter is binded
+    GetMethod get = httpGet("/notebook/interpreter/bind/" + note.id());
+    assertThat(get, isAllowed());
+    get.addRequestHeader("Origin", "http://localhost");
+    Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
+    }.getType());
+    List<Map<String, String>> body = (List<Map<String, String>>) resp.get("body");
+    assertTrue(0 < body.size());
+
+    get.releaseConnection();
+    //cleanup
+    ZeppelinServer.notebook.removeNote(note.getId());
+  }
+
+  @Test
+  public void testInterpreterRestart() throws IOException, InterruptedException {
+    // create new note
+    Note note = ZeppelinServer.notebook.createNote();
+    note.addParagraph();
+    Paragraph p = note.getLastParagraph();
+    Map config = p.getConfig();
+    config.put("enabled", true);
+
+    // run markdown paragraph
+    p.setConfig(config);
+    p.setText("%md markdown");
+    note.run(p.getId());
+    while (p.getStatus() != Status.FINISHED) {
+      Thread.sleep(100);
+    }
+    assertEquals("<p>markdown</p>\n", p.getResult().message());
+
+
+    // restart interpreter
+    for (InterpreterSetting setting : note.getNoteReplLoader().getInterpreterSettings()) {
+      if (setting.getName().equals("md")) {
+        // Call Restart Interpreter REST API
+        PutMethod put = httpPut("/interpreter/setting/restart/" + setting.id(), "");
+        assertThat("test interpreter restart:", put, isAllowed());
+        put.releaseConnection();
+        break;
+      }
+    }
+
+    // run markdown paragraph, again
+    p = note.addParagraph();
+    p.setConfig(config);
+    p.setText("%md markdown restarted");
+    note.run(p.getId());
+    while (p.getStatus() != Status.FINISHED) {
+      Thread.sleep(100);
+    }
+    assertEquals("<p>markdown restarted</p>\n", p.getResult().message());
+    //cleanup
+    ZeppelinServer.notebook.removeNote(note.getId());
+  }
+
+  @Test
+  public void testListRepository() throws IOException {
+    GetMethod get = httpGet("/interpreter/repository");
+    assertThat(get, isAllowed());
+    get.releaseConnection();
+  }
+
+  @Test
+  public void testAddDeleteRepository() throws IOException {
+    // Call create repository REST API
+    String repoId = "securecentral";
+    String jsonRequest = "{\"id\":\"" + repoId +
+        "\",\"url\":\"https://repo1.maven.org/maven2\",\"snapshot\":\"false\"}";
+
+    PostMethod post = httpPost("/interpreter/repository/", jsonRequest);
+    assertThat("Test create method:", post, isCreated());
+    post.releaseConnection();
+
+    // Call delete repository REST API
+    DeleteMethod delete = httpDelete("/interpreter/repository/" + repoId);
+    assertThat("Test delete method:", delete, isAllowed());
+    delete.releaseConnection();
+  }
+}

zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinRestApiTest.java

@@ -72,126 +72,6 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
     httpGetRoot.releaseConnection();
   }
 
-
-  @Test
-  public void getAvailableInterpreters() throws IOException {
-    // when
-    GetMethod get = httpGet("/interpreter");
-
-    // then
-    assertThat(get, isAllowed());
-    Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
-    }.getType());
-    Map<String, Object> body = (Map<String, Object>) resp.get("body");
-    assertEquals(ZeppelinServer.notebook.getInterpreterFactory().getRegisteredInterpreterList().size(), body.size());
-    get.releaseConnection();
-  }
-
-  @Test
-  public void getSettings() throws IOException {
-    // when
-    GetMethod get = httpGet("/interpreter/setting");
-
-    // then
-    Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
-    }.getType());
-    assertThat(get, isAllowed());
-    get.releaseConnection();
-  }
-
-  @Test
-  public void testSettingsCRUD() throws IOException {
-    // Call Create Setting REST API
-    String jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"propvalue\"}," +
-        "\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
-        "\"dependencies\":[]}";
-    PostMethod post = httpPost("/interpreter/setting/", jsonRequest);
-    LOG.info("testSettingCRUD create response\n" + post.getResponseBodyAsString());
-    assertThat("test create method:", post, isCreated());
-
-    Map<String, Object> resp = gson.fromJson(post.getResponseBodyAsString(), new TypeToken<Map<String, Object>>() {
-    }.getType());
-    Map<String, Object> body = (Map<String, Object>) resp.get("body");
-    //extract id from body string {id=2AWMQDNX7, name=md2, group=md,
-    String newSettingId =  body.toString().split(",")[0].split("=")[1];
-    post.releaseConnection();
-
-    // Call Update Setting REST API
-    jsonRequest = "{\"name\":\"md2\",\"group\":\"md\",\"properties\":{\"propname\":\"Otherpropvalue\"}," +
-        "\"interpreterGroup\":[{\"class\":\"org.apache.zeppelin.markdown.Markdown\",\"name\":\"md\"}]," +
-        "\"dependencies\":[]}";
-    PutMethod put = httpPut("/interpreter/setting/" + newSettingId, jsonRequest);
-    LOG.info("testSettingCRUD update response\n" + put.getResponseBodyAsString());
-    assertThat("test update method:", put, isAllowed());
-    put.releaseConnection();
-
-    // Call Delete Setting REST API
-    DeleteMethod delete = httpDelete("/interpreter/setting/" + newSettingId);
-    LOG.info("testSettingCRUD delete response\n" + delete.getResponseBodyAsString());
-    assertThat("Test delete method:", delete, isAllowed());
-    delete.releaseConnection();
-  }
-  @Test
-  public void testInterpreterAutoBinding() throws IOException {
-    // create note
-    Note note = ZeppelinServer.notebook.createNote();
-
-    // check interpreter is binded
-    GetMethod get = httpGet("/notebook/interpreter/bind/"+note.id());
-    assertThat(get, isAllowed());
-    get.addRequestHeader("Origin", "http://localhost");
-    Map<String, Object> resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken<Map<String, Object>>(){}.getType());
-    List<Map<String, String>> body = (List<Map<String, String>>) resp.get("body");
-    assertTrue(0 < body.size());
-
-    get.releaseConnection();
-    //cleanup
-    ZeppelinServer.notebook.removeNote(note.getId());
-  }
-
-  @Test
-  public void testInterpreterRestart() throws IOException, InterruptedException {
-    // create new note
-    Note note = ZeppelinServer.notebook.createNote();
-    note.addParagraph();
-    Paragraph p = note.getLastParagraph();
-    Map config = p.getConfig();
-    config.put("enabled", true);
-
-    // run markdown paragraph
-    p.setConfig(config);
-    p.setText("%md markdown");
-    note.run(p.getId());
-    while (p.getStatus() != Status.FINISHED) {
-      Thread.sleep(100);
-    }
-    assertEquals("<p>markdown</p>\n", p.getResult().message());
-
-    
-    // restart interpreter
-    for (InterpreterSetting setting : note.getNoteReplLoader().getInterpreterSettings()) {
-      if (setting.getName().equals("md")) {
-        // Call Restart Interpreter REST API
-        PutMethod put = httpPut("/interpreter/setting/restart/" + setting.id(), "");
-        assertThat("test interpreter restart:", put, isAllowed());
-        put.releaseConnection();
-        break;
-      }
-    }
-
-    // run markdown paragraph, again
-    p = note.addParagraph();
-    p.setConfig(config);
-    p.setText("%md markdown restarted");
-    note.run(p.getId());
-    while (p.getStatus() != Status.FINISHED) {
-      Thread.sleep(100);
-    }
-    assertEquals("<p>markdown restarted</p>\n", p.getResult().message());
-    //cleanup
-    ZeppelinServer.notebook.removeNote(note.getId());
-  }
-
   @Test
   public void testGetNotebookInfo() throws IOException {
     LOG.info("testGetNotebookInfo");
@@ -305,7 +185,7 @@ public class ZeppelinRestApiTest extends AbstractTestRestApi {
   }
 
   @Test
-  public void  testDeleteNote() throws IOException {
+  public void testDeleteNote() throws IOException {
     LOG.info("testDeleteNote");
     //Create note and get ID
     Note note = ZeppelinServer.notebook.createNote();