Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge branch 'master' into tachyon-interpreter

Browse source
This commit is contained in:
maocorte 2016-01-13 09:49:38 +01:00
commit 53011dc544
34 changed files with 762 additions and 221 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

77
.travis.yml
View file

@ -14,61 +14,46 @@
# limitations under the License.
language: java
jdk:
- oraclejdk7
matrix:
include:
# Test all modules
- jdk: "oraclejdk7"
env: SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr"
# Test spark module for 1.5.2
- jdk: "oraclejdk7"
env: SPARK_VER="1.5.2" HADOOP_VER="2.3" PROFILE="-Pspark-1.5 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
# Test spark module for 1.4.1
- jdk: "oraclejdk7"
env: SPARK_VER="1.4.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.4 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
# Test spark module for 1.3.1
- jdk: "oraclejdk7"
env: SPARK_VER="1.3.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.3 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
# Test spark module for 1.2.1
- jdk: "oraclejdk7"
env: SPARK_VER="1.2.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.2 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
# Test spark module for 1.1.1
- jdk: "oraclejdk7"
env: SPARK_VER="1.1.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.1 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify"
before_install:
- "export DISPLAY=:99.0"
- "sh -e /etc/init.d/xvfb start"
install:
- mvn package -DskipTests -Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding -B
- mvn $BUILD_FLAG $PROFILE -B
before_script:
-
- ./testing/startSparkCluster.sh $SPARK_VER $HADOOP_VER
- echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh
script:
# spark 1.6
- mvn package -Pbuild-distr -Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding -B
- ./testing/startSparkCluster.sh 1.6.0 2.3
- echo "export SPARK_HOME=`pwd`/spark-1.6.0-bin-hadoop2.3" > conf/zeppelin-env.sh
- mvn verify -Pusing-packaged-distr -Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pscalding -B
- ./testing/stopSparkCluster.sh 1.6.0 2.3
# spark 1.5
- rm -rf `pwd`/interpreter/spark
- mvn package -DskipTests -Pspark-1.5 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark'
- ./testing/startSparkCluster.sh 1.5.2 2.3
- echo "export SPARK_HOME=`pwd`/spark-1.5.2-bin-hadoop2.3" > conf/zeppelin-env.sh
- mvn package -Pspark-1.5 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false
- ./testing/stopSparkCluster.sh 1.5.2 2.3
# spark 1.4
- rm -rf `pwd`/interpreter/spark
- mvn package -DskipTests -Pspark-1.4 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark'
- ./testing/startSparkCluster.sh 1.4.1 2.3
- echo "export SPARK_HOME=`pwd`/spark-1.4.1-bin-hadoop2.3" > conf/zeppelin-env.sh
- mvn package -Pspark-1.4 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false
- ./testing/stopSparkCluster.sh 1.4.1 2.3
# spark 1.3
- rm -rf `pwd`/interpreter/spark
- mvn package -DskipTests -Pspark-1.3 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark'
- ./testing/startSparkCluster.sh 1.3.1 2.3
- echo "export SPARK_HOME=`pwd`/spark-1.3.1-bin-hadoop2.3" > conf/zeppelin-env.sh
- mvn package -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false
- ./testing/stopSparkCluster.sh 1.3.1 2.3
# spark 1.2
- rm -rf `pwd`/interpreter/spark
- mvn package -Pspark-1.2 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark'
- ./testing/startSparkCluster.sh 1.2.1 2.3
- echo "export SPARK_HOME=`pwd`/spark-1.2.1-bin-hadoop2.3" > conf/zeppelin-env.sh
- mvn package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false
- ./testing/stopSparkCluster.sh 1.2.1 2.3
# spark 1.1
- rm -rf `pwd`/interpreter/spark
- mvn package -Pspark-1.1 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark'
- ./testing/startSparkCluster.sh 1.1.1 2.3
- echo "export SPARK_HOME=`pwd`/spark-1.1.1-bin-hadoop2.3" > conf/zeppelin-env.sh
- mvn package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false
- ./testing/stopSparkCluster.sh 1.1.1 2.3
- mvn $TEST_FLAG $PROFILE -B
after_failure:
- cat target/rat.txt
@ -77,7 +62,7 @@ after_failure:
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out
after_script:
-
- ./testing/stopSparkCluster.sh $SPARK_VER $HADOOP_VER
notifications:
slack:

16
CONTRIBUTING.md
View file

@ -68,7 +68,8 @@ Here are some things you will need to build and test Zeppelin.
### Software Configuration Management (SCM)
Zeppelin uses Git for its SCM system. Hosted by github.com. `https://github.com/apache/incubator-zeppelin` you'll need git client installed in your development machine.
Zeppelin uses Git for its SCM system. `http://git.apache.org/incubator-zeppelin.git` you'll need git client installed in your development machine.
For write access, `https://git-wip-us.apache.org/repos/asf/incubator-zeppelin.git`
### Integrated Development Environment (IDE)
@ -114,26 +115,31 @@ To build the code, install
* Apache Maven
## Getting the source code
First of all, you need the Zeppelin source code. The official location for Zeppelin is [https://github.com/apache/incubator-zeppelin](https://github.com/apache/incubator-zeppelin)
First of all, you need the Zeppelin source code. The official location for Zeppelin is [http://git.apache.org/incubator-zeppelin.git](http://git.apache.org/incubator-zeppelin.git).
### git access
Get the source code on your development machine using git.
```
git clone git@github.com:apache/incubator-zeppelin.git zeppelin
git clone http://git.apache.org/incubator-zeppelin.git zeppelin
```
You may also want to develop against a specific release. For example, for branch-0.1
```
git clone -b branch-0.1 git@github.com:apache/incubator-zeppelin.git zeppelin
git clone -b branch-0.1 http://git.apache.org/incubator-zeppelin.git zeppelin
```
or with write access
```
git clone https://git-wip-us.apache.org/repos/asf/incubator-zeppelin.git
```
### Fork repository
If you want not only build Zeppelin but also make change, then you need fork Zeppelin repository and make pull request.
If you want not only build Zeppelin but also make change, then you need fork Zeppelin github mirror repository (https://github.com/apache/incubator-zeppelin) and make pull request.
## Build

2
dev/create_release.sh
View file

@ -125,7 +125,7 @@ function make_binary_release() {
rm -rf ${WORKING_DIR}/zeppelin-${RELEASE_NAME}-bin-${BIN_RELEASE_NAME}
}
make_binary_release all "-Pspark-1.5 -Phadoop-2.4 -Pyarn -Ppyspark"
make_binary_release all "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark"
# remove non release files and dirs
rm -rf ${WORKING_DIR}/zeppelin

2
docs/index.md
View file

@ -117,7 +117,7 @@ This way, you can easily embed it as an iframe inside of your website.</p>
<br />
### 100% Opensource
Apache Zeppelin (incubating) is Apache2 Licensed software. Please check out the [source repository](https://github.com/apache/incubator-zeppelin) and [How to contribute](./development/howtocontribute.html)
Apache Zeppelin (incubating) is Apache2 Licensed software. Please check out the [source repository](http://git.apache.org/incubator-zeppelin.git) and [How to contribute](./development/howtocontribute.html)
Zeppelin has a very active development community.
Join the [Mailing list](./community.html) and report issues on our [Issue tracker](https://issues.apache.org/jira/browse/ZEPPELIN).

7
ignite/src/main/resources/default-ignite-jdbc.xml
View file

@ -33,13 +33,6 @@
<property name="peerClassLoadingEnabled" value="true"/>
<property name="marshaller">
<bean class="org.apache.ignite.marshaller.optimized.OptimizedMarshaller">
<!-- Set to false to allow non-serializable objects in examples, default is true. -->
<property name="requireSerializable" value="false"/>
</bean>
</property>
<!-- Explicitly configure TCP discovery SPI to provide list of initial nodes. -->
<property name="discoverySpi">
<bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">

1
ignite/src/test/java/org/apache/zeppelin/ignite/IgniteSqlInterpreterTest.java
View file

@ -60,7 +60,6 @@ public class IgniteSqlInterpreterTest {
IgniteConfiguration cfg = new IgniteConfiguration();
cfg.setDiscoverySpi(discoSpi);
cfg.setPeerClassLoadingEnabled(true);
cfg.setMarshaller(new OptimizedMarshaller());
cfg.setGridName("test");

36
spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java
View file

@ -40,7 +40,7 @@ import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.WrappedInterpreter;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.spark.dep.DependencyContext;
import org.apache.zeppelin.spark.dep.SparkDependencyContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonatype.aether.resolution.ArtifactResolutionException;
@ -69,7 +69,9 @@ public class DepInterpreter extends Interpreter {
"spark",
DepInterpreter.class.getName(),
new InterpreterPropertyBuilder()
.add("zeppelin.dep.localrepo", "local-repo", "local repository for dependency loader")
.add("zeppelin.dep.localrepo",
getSystemDefault("ZEPPELIN_DEP_LOCALREPO", null, "local-repo"),
"local repository for dependency loader")
.add("zeppelin.dep.additionalRemoteRepository",
"spark-packages,http://dl.bintray.com/spark-packages/maven,false;",
"A list of 'id,remote-repository-URL,is-snapshot;' for each remote repository.")
@ -79,7 +81,7 @@ public class DepInterpreter extends Interpreter {
private SparkIMain intp;
private ByteArrayOutputStream out;
private DependencyContext depc;
private SparkDependencyContext depc;
private SparkJLineCompletion completor;
private SparkILoop interpreter;
static final Logger LOGGER = LoggerFactory.getLogger(DepInterpreter.class);
@ -88,10 +90,30 @@ public class DepInterpreter extends Interpreter {
super(property);
}
public DependencyContext getDependencyContext() {
public SparkDependencyContext getDependencyContext() {
return depc;
}
public static String getSystemDefault(
String envName,
String propertyName,
String defaultValue) {
if (envName != null && !envName.isEmpty()) {
String envValue = System.getenv().get(envName);
if (envValue != null) {
return envValue;
}
}
if (propertyName != null && !propertyName.isEmpty()) {
String propValue = System.getProperty(propertyName);
if (propValue != null) {
return propValue;
}
}
return defaultValue;
}
@Override
public void close() {
@ -152,16 +174,16 @@ public class DepInterpreter extends Interpreter {
intp.setContextClassLoader();
intp.initializeSynchronous();
depc = new DependencyContext(getProperty("zeppelin.dep.localrepo"),
depc = new SparkDependencyContext(getProperty("zeppelin.dep.localrepo"),
getProperty("zeppelin.dep.additionalRemoteRepository"));
completor = new SparkJLineCompletion(intp);
intp.interpret("@transient var _binder = new java.util.HashMap[String, Object]()");
Map<String, Object> binder = (Map<String, Object>) getValue("_binder");
binder.put("depc", depc);
intp.interpret("@transient val z = "
+ "_binder.get(\"depc\").asInstanceOf[org.apache.zeppelin.spark.dep.DependencyContext]");
+ "_binder.get(\"depc\")"
+ ".asInstanceOf[org.apache.zeppelin.spark.dep.SparkDependencyContext]");
}

4
spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
View file

@ -54,7 +54,7 @@ import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.InterpreterResult.Code;
import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
import org.apache.zeppelin.interpreter.WrappedInterpreter;
import org.apache.zeppelin.spark.dep.DependencyContext;
import org.apache.zeppelin.spark.dep.SparkDependencyContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -127,7 +127,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand
URL [] urls = new URL[0];
if (depInterpreter != null) {
DependencyContext depc = depInterpreter.getDependencyContext();
SparkDependencyContext depc = depInterpreter.getDependencyContext();
if (depc != null) {
List<File> files = depc.getFiles();
List<URL> urlList = new LinkedList<URL>();

14
spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
View file

@ -55,8 +55,8 @@ import org.apache.zeppelin.interpreter.InterpreterUtils;
import org.apache.zeppelin.interpreter.WrappedInterpreter;
import org.apache.zeppelin.scheduler.Scheduler;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.apache.zeppelin.spark.dep.DependencyContext;
import org.apache.zeppelin.spark.dep.DependencyResolver;
import org.apache.zeppelin.spark.dep.SparkDependencyContext;
import org.apache.zeppelin.spark.dep.SparkDependencyResolver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -117,7 +117,7 @@ public class SparkInterpreter extends Interpreter {
private SparkContext sc;
private ByteArrayOutputStream out;
private SQLContext sqlc;
private DependencyResolver dep;
private SparkDependencyResolver dep;
private SparkJLineCompletion completor;
private JobProgressListener sparkListener;
@ -222,9 +222,9 @@ public class SparkInterpreter extends Interpreter {
return sqlc;
}
public DependencyResolver getDependencyResolver() {
public SparkDependencyResolver getDependencyResolver() {
if (dep == null) {
dep = new DependencyResolver(intp,
dep = new SparkDependencyResolver(intp,
sc,
getProperty("zeppelin.dep.localrepo"),
getProperty("zeppelin.dep.additionalRemoteRepository"));
@ -427,7 +427,7 @@ public class SparkInterpreter extends Interpreter {
// add dependency from DepInterpreter
DepInterpreter depInterpreter = getDepInterpreter();
if (depInterpreter != null) {
DependencyContext depc = depInterpreter.getDependencyContext();
SparkDependencyContext depc = depInterpreter.getDependencyContext();
if (depc != null) {
List<File> files = depc.getFiles();
if (files != null) {
@ -536,7 +536,7 @@ public class SparkInterpreter extends Interpreter {
// add jar
if (depInterpreter != null) {
DependencyContext depc = depInterpreter.getDependencyContext();
SparkDependencyContext depc = depInterpreter.getDependencyContext();
if (depc != null) {
List<File> files = depc.getFilesDist();
if (files != null) {

6
spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java
View file

@ -43,7 +43,7 @@ import org.apache.zeppelin.display.Input.ParamOption;
import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterContextRunner;
import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.spark.dep.DependencyResolver;
import org.apache.zeppelin.spark.dep.SparkDependencyResolver;
import scala.Tuple2;
import scala.Unit;
@ -53,14 +53,14 @@ import scala.collection.Iterable;
* Spark context for zeppelin.
*/
public class ZeppelinContext extends HashMap<String, Object> {
private DependencyResolver dep;
private SparkDependencyResolver dep;
private PrintStream out;
private InterpreterContext interpreterContext;
private int maxResult;
public ZeppelinContext(SparkContext sc, SQLContext sql,
InterpreterContext interpreterContext,
DependencyResolver dep, PrintStream printStream,
SparkDependencyResolver dep, PrintStream printStream,
int maxResult) {
this.sc = sc;
this.sqlContext = sql;

181
spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyContext.java Normal file
View file

@ -0,0 +1,181 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.spark.dep;
import java.io.File;
import java.net.MalformedURLException;
import java.util.LinkedList;
import java.util.List;
import org.apache.zeppelin.dep.Booter;
import org.apache.zeppelin.dep.Dependency;
import org.apache.zeppelin.dep.Repository;
import org.sonatype.aether.RepositorySystem;
import org.sonatype.aether.RepositorySystemSession;
import org.sonatype.aether.artifact.Artifact;
import org.sonatype.aether.collection.CollectRequest;
import org.sonatype.aether.graph.DependencyFilter;
import org.sonatype.aether.repository.RemoteRepository;
import org.sonatype.aether.repository.Authentication;
import org.sonatype.aether.resolution.ArtifactResolutionException;
import org.sonatype.aether.resolution.ArtifactResult;
import org.sonatype.aether.resolution.DependencyRequest;
import org.sonatype.aether.resolution.DependencyResolutionException;
import org.sonatype.aether.util.artifact.DefaultArtifact;
import org.sonatype.aether.util.artifact.JavaScopes;
import org.sonatype.aether.util.filter.DependencyFilterUtils;
import org.sonatype.aether.util.filter.PatternExclusionsDependencyFilter;
/**
*
*/
public class SparkDependencyContext {
List<Dependency> dependencies = new LinkedList<Dependency>();
List<Repository> repositories = new LinkedList<Repository>();
List<File> files = new LinkedList<File>();
List<File> filesDist = new LinkedList<File>();
private RepositorySystem system = Booter.newRepositorySystem();
private RepositorySystemSession session;
private RemoteRepository mavenCentral = Booter.newCentralRepository();
private RemoteRepository mavenLocal = Booter.newLocalRepository();
private List<RemoteRepository> additionalRepos = new LinkedList<RemoteRepository>();
public SparkDependencyContext(String localRepoPath, String additionalRemoteRepository) {
session = Booter.newRepositorySystemSession(system, localRepoPath);
addRepoFromProperty(additionalRemoteRepository);
}
public Dependency load(String lib) {
Dependency dep = new Dependency(lib);
if (dependencies.contains(dep)) {
dependencies.remove(dep);
}
dependencies.add(dep);
return dep;
}
public Repository addRepo(String name) {
Repository rep = new Repository(name);
repositories.add(rep);
return rep;
}
public void reset() {
dependencies = new LinkedList<Dependency>();
repositories = new LinkedList<Repository>();
files = new LinkedList<File>();
filesDist = new LinkedList<File>();
}
private void addRepoFromProperty(String listOfRepo) {
if (listOfRepo != null) {
String[] repos = listOfRepo.split(";");
for (String repo : repos) {
String[] parts = repo.split(",");
if (parts.length == 3) {
String id = parts[0].trim();
String url = parts[1].trim();
boolean isSnapshot = Boolean.parseBoolean(parts[2].trim());
if (id.length() > 1 && url.length() > 1) {
RemoteRepository rr = new RemoteRepository(id, "default", url);
rr.setPolicy(isSnapshot, null);
additionalRepos.add(rr);
}
}
}
}
}
/**
* fetch all artifacts
* @return
* @throws MalformedURLException
* @throws ArtifactResolutionException
* @throws DependencyResolutionException
*/
public List<File> fetch() throws MalformedURLException,
DependencyResolutionException, ArtifactResolutionException {
for (Dependency dep : dependencies) {
if (!dep.isLocalFsArtifact()) {
List<ArtifactResult> artifacts = fetchArtifactWithDep(dep);
for (ArtifactResult artifact : artifacts) {
if (dep.isDist()) {
filesDist.add(artifact.getArtifact().getFile());
}
files.add(artifact.getArtifact().getFile());
}
} else {
if (dep.isDist()) {
filesDist.add(new File(dep.getGroupArtifactVersion()));
}
files.add(new File(dep.getGroupArtifactVersion()));
}
}
return files;
}
private List<ArtifactResult> fetchArtifactWithDep(Dependency dep)
throws DependencyResolutionException, ArtifactResolutionException {
Artifact artifact = new DefaultArtifact(
SparkDependencyResolver.inferScalaVersion(dep.getGroupArtifactVersion()));
DependencyFilter classpathFlter = DependencyFilterUtils
.classpathFilter(JavaScopes.COMPILE);
PatternExclusionsDependencyFilter exclusionFilter = new PatternExclusionsDependencyFilter(
SparkDependencyResolver.inferScalaVersion(dep.getExclusions()));
CollectRequest collectRequest = new CollectRequest();
collectRequest.setRoot(new org.sonatype.aether.graph.Dependency(artifact,
JavaScopes.COMPILE));
collectRequest.addRepository(mavenCentral);
collectRequest.addRepository(mavenLocal);
for (RemoteRepository repo : additionalRepos) {
collectRequest.addRepository(repo);
}
for (Repository repo : repositories) {
RemoteRepository rr = new RemoteRepository(repo.getName(), "default", repo.getUrl());
rr.setPolicy(repo.isSnapshot(), null);
Authentication auth = repo.getAuthentication();
if (auth != null) {
rr.setAuthentication(auth);
}
collectRequest.addRepository(rr);
}
DependencyRequest dependencyRequest = new DependencyRequest(collectRequest,
DependencyFilterUtils.andFilter(exclusionFilter, classpathFlter));
return system.resolveDependencies(session, dependencyRequest).getArtifactResults();
}
public List<File> getFiles() {
return files;
}
public List<File> getFilesDist() {
return filesDist;
}
}

52
spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyResolver.java → spark/src/main/java/org/apache/zeppelin/spark/dep/SparkDependencyResolver.java
View file

@ -30,10 +30,9 @@ import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.spark.SparkContext;
import org.apache.spark.repl.SparkIMain;
import org.apache.zeppelin.dep.AbstractDependencyResolver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonatype.aether.RepositorySystem;
import org.sonatype.aether.RepositorySystemSession;
import org.sonatype.aether.artifact.Artifact;
import org.sonatype.aether.collection.CollectRequest;
import org.sonatype.aether.graph.Dependency;
@ -56,21 +55,13 @@ import scala.tools.nsc.util.MergedClassPath;
/**
* Deps resolver.
* Add new dependencies from mvn repo (at runetime) to Zeppelin.
* Add new dependencies from mvn repo (at runtime) to Spark interpreter group.
*/
public class DependencyResolver {
Logger logger = LoggerFactory.getLogger(DependencyResolver.class);
public class SparkDependencyResolver extends AbstractDependencyResolver {
Logger logger = LoggerFactory.getLogger(SparkDependencyResolver.class);
private Global global;
private SparkIMain intp;
private SparkContext sc;
private RepositorySystem system = Booter.newRepositorySystem();
private List<RemoteRepository> repos = new LinkedList<RemoteRepository>();
private RepositorySystemSession session;
private DependencyFilter classpathFlter = DependencyFilterUtils.classpathFilter(
JavaScopes.COMPILE,
JavaScopes.PROVIDED,
JavaScopes.RUNTIME,
JavaScopes.SYSTEM);
private final String[] exclusions = new String[] {"org.scala-lang:scala-library",
"org.scala-lang:scala-compiler",
@ -80,40 +71,15 @@ public class DependencyResolver {
"org.apache.zeppelin:zeppelin-spark",
"org.apache.zeppelin:zeppelin-server"};
public DependencyResolver(SparkIMain intp, SparkContext sc, String localRepoPath,
public SparkDependencyResolver(SparkIMain intp, SparkContext sc, String localRepoPath,
String additionalRemoteRepository) {
super(localRepoPath);
this.intp = intp;
this.global = intp.global();
this.sc = sc;
session = Booter.newRepositorySystemSession(system, localRepoPath);
repos.add(Booter.newCentralRepository()); // add maven central
repos.add(Booter.newLocalRepository());
addRepoFromProperty(additionalRemoteRepository);
}
public void addRepo(String id, String url, boolean snapshot) {
synchronized (repos) {
delRepo(id);
RemoteRepository rr = new RemoteRepository(id, "default", url);
rr.setPolicy(snapshot, null);
repos.add(rr);
}
}
public RemoteRepository delRepo(String id) {
synchronized (repos) {
Iterator<RemoteRepository> it = repos.iterator();
if (it.hasNext()) {
RemoteRepository repo = it.next();
if (repo.getId().equals(id)) {
it.remove();
return repo;
}
}
}
return null;
}
private void addRepoFromProperty(String listOfRepo) {
if (listOfRepo != null) {
String[] repos = listOfRepo.split(";");
@ -309,16 +275,16 @@ public class DependencyResolver {
}
/**
*
* @param dependency
* @param excludes list of pattern can either be of the form groupId:artifactId
* @return
* @throws Exception
*/
@Override
public List<ArtifactResult> getArtifactsWithDep(String dependency,
Collection<String> excludes) throws Exception {
Artifact artifact = new DefaultArtifact(inferScalaVersion(dependency));
DependencyFilter classpathFlter = DependencyFilterUtils.classpathFilter( JavaScopes.COMPILE );
DependencyFilter classpathFilter = DependencyFilterUtils.classpathFilter(JavaScopes.COMPILE);
PatternExclusionsDependencyFilter exclusionFilter =
new PatternExclusionsDependencyFilter(inferScalaVersion(excludes));
@ -331,7 +297,7 @@ public class DependencyResolver {
}
}
DependencyRequest dependencyRequest = new DependencyRequest(collectRequest,
DependencyFilterUtils.andFilter(exclusionFilter, classpathFlter));
DependencyFilterUtils.andFilter(exclusionFilter, classpathFilter));
return system.resolveDependencies(session, dependencyRequest).getArtifactResults();
}

22
spark/src/test/java/org/apache/zeppelin/spark/dep/DependencyResolverTest.java → spark/src/test/java/org/apache/zeppelin/spark/dep/SparkDependencyResolverTest.java
View file

@ -19,10 +19,10 @@ package org.apache.zeppelin.spark.dep;
import static org.junit.Assert.assertEquals;
import org.apache.zeppelin.spark.dep.DependencyResolver;
import org.apache.zeppelin.spark.dep.SparkDependencyResolver;
import org.junit.Test;
public class DependencyResolverTest {
public class SparkDependencyResolverTest {
@Test
public void testInferScalaVersion() {
@ -30,23 +30,23 @@ public class DependencyResolverTest {
String scalaVersion = version[0] + "." + version[1];
assertEquals("groupId:artifactId:version",
DependencyResolver.inferScalaVersion("groupId:artifactId:version"));
SparkDependencyResolver.inferScalaVersion("groupId:artifactId:version"));
assertEquals("groupId:artifactId_" + scalaVersion + ":version",
DependencyResolver.inferScalaVersion("groupId::artifactId:version"));
SparkDependencyResolver.inferScalaVersion("groupId::artifactId:version"));
assertEquals("groupId:artifactId:version::test",
DependencyResolver.inferScalaVersion("groupId:artifactId:version::test"));
SparkDependencyResolver.inferScalaVersion("groupId:artifactId:version::test"));
assertEquals("*",
DependencyResolver.inferScalaVersion("*"));
SparkDependencyResolver.inferScalaVersion("*"));
assertEquals("groupId:*",
DependencyResolver.inferScalaVersion("groupId:*"));
SparkDependencyResolver.inferScalaVersion("groupId:*"));
assertEquals("groupId:artifactId*",
DependencyResolver.inferScalaVersion("groupId:artifactId*"));
SparkDependencyResolver.inferScalaVersion("groupId:artifactId*"));
assertEquals("groupId:artifactId_" + scalaVersion,
DependencyResolver.inferScalaVersion("groupId::artifactId"));
SparkDependencyResolver.inferScalaVersion("groupId::artifactId"));
assertEquals("groupId:artifactId_" + scalaVersion + "*",
DependencyResolver.inferScalaVersion("groupId::artifactId*"));
SparkDependencyResolver.inferScalaVersion("groupId::artifactId*"));
assertEquals("groupId:artifactId_" + scalaVersion + ":*",
DependencyResolver.inferScalaVersion("groupId::artifactId:*"));
SparkDependencyResolver.inferScalaVersion("groupId::artifactId:*"));
}
}

117
zeppelin-interpreter/pom.xml
View file

@ -97,6 +97,123 @@
<artifactId>commons-lang3</artifactId>
<version>${commons-lang.version}</version>
</dependency>
<!-- Aether :: maven dependency resolution -->
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-plugin-api</artifactId>
<version>3.0</version>
<exclusions>
<exclusion>
<groupId>org.codehaus.plexus</groupId>
<artifactId>plexus-utils</artifactId>
</exclusion>
<exclusion>
<groupId>org.sonatype.sisu</groupId>
<artifactId>sisu-inject-plexus</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.maven</groupId>
<artifactId>maven-model</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-api</artifactId>
<version>1.12</version>
</dependency>
<dependency>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-util</artifactId>
<version>1.12</version>
</dependency>
<dependency>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-impl</artifactId>
<version>1.12</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-aether-provider</artifactId>
<version>3.0.3</version>
<exclusions>
<exclusion>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-spi</artifactId>
</exclusion>
<exclusion>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-util</artifactId>
</exclusion>
<exclusion>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-impl</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.plexus</groupId>
<artifactId>plexus-utils</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-connector-file</artifactId>
<version>1.12</version>
</dependency>
<dependency>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-connector-wagon</artifactId>
<version>1.12</version>
<exclusions>
<exclusion>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-provider-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-provider-api</artifactId>
<version>1.0</version>
<exclusions>
<exclusion>
<groupId>org.codehaus.plexus</groupId>
<artifactId>plexus-utils</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-http-lightweight</artifactId>
<version>1.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-http-shared</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-http</artifactId>
<version>1.0</version>
<exclusions>
</exclusions>
</dependency>
</dependencies>
<build>

70
zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java Normal file
View file

@ -0,0 +1,70 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.dep;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import org.sonatype.aether.RepositorySystem;
import org.sonatype.aether.RepositorySystemSession;
import org.sonatype.aether.repository.RemoteRepository;
import org.sonatype.aether.resolution.ArtifactResult;
/**
* Abstract dependency resolver.
* Add new dependencies from mvn repo (at runtime) Zeppelin.
*/
public abstract class AbstractDependencyResolver {
protected RepositorySystem system = Booter.newRepositorySystem();
protected List<RemoteRepository> repos = new LinkedList<RemoteRepository>();
protected RepositorySystemSession session;
public AbstractDependencyResolver(String localRepoPath) {
session = Booter.newRepositorySystemSession(system, localRepoPath);
repos.add(Booter.newCentralRepository()); // add maven central
repos.add(Booter.newLocalRepository());
}
public void addRepo(String id, String url, boolean snapshot) {
synchronized (repos) {
delRepo(id);
RemoteRepository rr = new RemoteRepository(id, "default", url);
rr.setPolicy(snapshot, null);
repos.add(rr);
}
}
public RemoteRepository delRepo(String id) {
synchronized (repos) {
Iterator<RemoteRepository> it = repos.iterator();
if (it.hasNext()) {
RemoteRepository repo = it.next();
if (repo.getId().equals(id)) {
it.remove();
return repo;
}
}
}
return null;
}
public abstract List<ArtifactResult> getArtifactsWithDep(String dependency,
Collection<String> excludes) throws Exception;
}

4
spark/src/main/java/org/apache/zeppelin/spark/dep/Booter.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java
View file

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.zeppelin.spark.dep;
package org.apache.zeppelin.dep;
import java.io.File;
@ -64,7 +64,7 @@ public class Booter {
public static RemoteRepository newCentralRepository() {
return new RemoteRepository("central", "default", "http://repo1.maven.org/maven2/");
}
public static RemoteRepository newLocalRepository() {
return new RemoteRepository("local",
"default", "file://" + System.getProperty("user.home") + "/.m2/repository");

2
spark/src/main/java/org/apache/zeppelin/spark/dep/Dependency.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Dependency.java
View file

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.zeppelin.spark.dep;
package org.apache.zeppelin.dep;
import java.util.LinkedList;
import java.util.List;

41
spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyContext.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyContext.java
View file

@ -15,14 +15,12 @@
* limitations under the License.
*/
package org.apache.zeppelin.spark.dep;
package org.apache.zeppelin.dep;
import java.io.File;
import java.net.MalformedURLException;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
import org.sonatype.aether.RepositorySystem;
import org.sonatype.aether.RepositorySystemSession;
@ -30,7 +28,6 @@ import org.sonatype.aether.artifact.Artifact;
import org.sonatype.aether.collection.CollectRequest;
import org.sonatype.aether.graph.DependencyFilter;
import org.sonatype.aether.repository.RemoteRepository;
import org.sonatype.aether.repository.Authentication;
import org.sonatype.aether.resolution.ArtifactResolutionException;
import org.sonatype.aether.resolution.ArtifactResult;
import org.sonatype.aether.resolution.DependencyRequest;
@ -54,11 +51,9 @@ public class DependencyContext {
private RepositorySystemSession session;
private RemoteRepository mavenCentral = Booter.newCentralRepository();
private RemoteRepository mavenLocal = Booter.newLocalRepository();
private List<RemoteRepository> additionalRepos = new LinkedList<RemoteRepository>();
public DependencyContext(String localRepoPath, String additionalRemoteRepository) {
session = Booter.newRepositorySystemSession(system, localRepoPath);
addRepoFromProperty(additionalRemoteRepository);
public DependencyContext(String localRepoPath) {
session = Booter.newRepositorySystemSession(system, localRepoPath);
}
public Dependency load(String lib) {
@ -85,24 +80,6 @@ public class DependencyContext {
filesDist = new LinkedList<File>();
}
private void addRepoFromProperty(String listOfRepo) {
if (listOfRepo != null) {
String[] repos = listOfRepo.split(";");
for (String repo : repos) {
String[] parts = repo.split(",");
if (parts.length == 3) {
String id = parts[0].trim();
String url = parts[1].trim();
boolean isSnapshot = Boolean.parseBoolean(parts[2].trim());
if (id.length() > 1 && url.length() > 1) {
RemoteRepository rr = new RemoteRepository(id, "default", url);
rr.setPolicy(isSnapshot, null);
additionalRepos.add(rr);
}
}
}
}
}
/**
* fetch all artifacts
@ -136,13 +113,12 @@ public class DependencyContext {
private List<ArtifactResult> fetchArtifactWithDep(Dependency dep)
throws DependencyResolutionException, ArtifactResolutionException {
Artifact artifact = new DefaultArtifact(
DependencyResolver.inferScalaVersion(dep.getGroupArtifactVersion()));
Artifact artifact = new DefaultArtifact(dep.getGroupArtifactVersion());
DependencyFilter classpathFlter = DependencyFilterUtils
.classpathFilter(JavaScopes.COMPILE);
PatternExclusionsDependencyFilter exclusionFilter = new PatternExclusionsDependencyFilter(
DependencyResolver.inferScalaVersion(dep.getExclusions()));
dep.getExclusions());
CollectRequest collectRequest = new CollectRequest();
collectRequest.setRoot(new org.sonatype.aether.graph.Dependency(artifact,
@ -150,16 +126,9 @@ public class DependencyContext {
collectRequest.addRepository(mavenCentral);
collectRequest.addRepository(mavenLocal);
for (RemoteRepository repo : additionalRepos) {
collectRequest.addRepository(repo);
}
for (Repository repo : repositories) {
RemoteRepository rr = new RemoteRepository(repo.getName(), "default", repo.getUrl());
rr.setPolicy(repo.isSnapshot(), null);
Authentication auth = repo.getAuthentication();
if (auth != null) {
rr.setAuthentication(auth);
}
collectRequest.addRepository(rr);
}

163
zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java Normal file
View file

@ -0,0 +1,163 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.dep;
import java.io.File;
import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sonatype.aether.artifact.Artifact;
import org.sonatype.aether.collection.CollectRequest;
import org.sonatype.aether.graph.Dependency;
import org.sonatype.aether.graph.DependencyFilter;
import org.sonatype.aether.repository.RemoteRepository;
import org.sonatype.aether.resolution.ArtifactResult;
import org.sonatype.aether.resolution.DependencyRequest;
import org.sonatype.aether.util.artifact.DefaultArtifact;
import org.sonatype.aether.util.artifact.JavaScopes;
import org.sonatype.aether.util.filter.DependencyFilterUtils;
import org.sonatype.aether.util.filter.PatternExclusionsDependencyFilter;
/**
* Deps resolver.
* Add new dependencies from mvn repo (at runtime) to Zeppelin.
*/
public class DependencyResolver extends AbstractDependencyResolver {
Logger logger = LoggerFactory.getLogger(DependencyResolver.class);
private final String[] exclusions = new String[] {"org.apache.zeppelin:zeppelin-zengine",
"org.apache.zeppelin:zeppelin-interpreter",
"org.apache.zeppelin:zeppelin-server"};
public DependencyResolver(String localRepoPath) {
super(localRepoPath);
}
public List<File> load(String artifact) throws Exception {
return load(artifact, new LinkedList<String>());
}
public List<File> load(String artifact, String destPath) throws Exception {
return load(artifact, new LinkedList<String>(), destPath);
}
public synchronized List<File> load(String artifact, Collection<String> excludes)
throws Exception {
if (StringUtils.isBlank(artifact)) {
// Should throw here
throw new RuntimeException("Invalid artifact to load");
}
// <groupId>:<artifactId>[:<extension>[:<classifier>]]:<version>
int numSplits = artifact.split(":").length;
if (numSplits >= 3 && numSplits <= 6) {
return loadFromMvn(artifact, excludes);
} else {
LinkedList<File> libs = new LinkedList<File>();
libs.add(new File(artifact));
return libs;
}
}
public List<File> load(String artifact, Collection<String> excludes, String destPath)
throws Exception {
List<File> libs = load(artifact, excludes);
// find home dir
String home = System.getenv("ZEPPELIN_HOME");
if (home == null) {
home = System.getProperty("zeppelin.home");
}
if (home == null) {
home = "..";
}
for (File srcFile: libs) {
File destFile = new File(home + "/" + destPath, srcFile.getName());
if (!destFile.exists() || !FileUtils.contentEquals(srcFile, destFile)) {
FileUtils.copyFile(srcFile, destFile);
logger.info("copy {} to {}", srcFile.getAbsolutePath(), destPath);
}
}
return libs;
}
private List<File> loadFromMvn(String artifact, Collection<String> excludes) throws Exception {
Collection<String> allExclusions = new LinkedList<String>();
allExclusions.addAll(excludes);
allExclusions.addAll(Arrays.asList(exclusions));
List<ArtifactResult> listOfArtifact;
listOfArtifact = getArtifactsWithDep(artifact, allExclusions);
Iterator<ArtifactResult> it = listOfArtifact.iterator();
while (it.hasNext()) {
Artifact a = it.next().getArtifact();
String gav = a.getGroupId() + ":" + a.getArtifactId() + ":" + a.getVersion();
for (String exclude : allExclusions) {
if (gav.startsWith(exclude)) {
it.remove();
break;
}
}
}
List<File> files = new LinkedList<File>();
for (ArtifactResult artifactResult : listOfArtifact) {
files.add(artifactResult.getArtifact().getFile());
logger.info("load {}", artifactResult.getArtifact().getFile().getAbsolutePath());
}
return files;
}
/**
* @param dependency
* @param excludes list of pattern can either be of the form groupId:artifactId
* @return
* @throws Exception
*/
@Override
public List<ArtifactResult> getArtifactsWithDep(String dependency,
Collection<String> excludes) throws Exception {
Artifact artifact = new DefaultArtifact(dependency);
DependencyFilter classpathFilter = DependencyFilterUtils.classpathFilter(JavaScopes.COMPILE);
PatternExclusionsDependencyFilter exclusionFilter =
new PatternExclusionsDependencyFilter(excludes);
CollectRequest collectRequest = new CollectRequest();
collectRequest.setRoot(new Dependency(artifact, JavaScopes.COMPILE));
synchronized (repos) {
for (RemoteRepository repo : repos) {
collectRequest.addRepository(repo);
}
}
DependencyRequest dependencyRequest = new DependencyRequest(collectRequest,
DependencyFilterUtils.andFilter(exclusionFilter, classpathFilter));
return system.resolveDependencies(session, dependencyRequest).getArtifactResults();
}
}

4
spark/src/main/java/org/apache/zeppelin/spark/dep/Repository.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Repository.java
View file

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.zeppelin.spark.dep;
package org.apache.zeppelin.dep;
import org.sonatype.aether.repository.Authentication;
/**
*
@ -70,7 +70,7 @@ public class Repository {
return this;
}
protected Authentication getAuthentication() {
public Authentication getAuthentication() {
Authentication auth = null;
if (this.username != null && this.password != null) {
auth = new Authentication(this.username, this.password);

2
spark/src/main/java/org/apache/zeppelin/spark/dep/RepositoryListener.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/RepositoryListener.java
View file

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.zeppelin.spark.dep;
package org.apache.zeppelin.dep;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

2
spark/src/main/java/org/apache/zeppelin/spark/dep/RepositorySystemFactory.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/RepositorySystemFactory.java
View file

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.zeppelin.spark.dep;
package org.apache.zeppelin.dep;
import org.apache.maven.repository.internal.DefaultServiceLocator;
import org.apache.maven.wagon.Wagon;

2
spark/src/main/java/org/apache/zeppelin/spark/dep/TransferListener.java → zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/TransferListener.java
View file

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.zeppelin.spark.dep;
package org.apache.zeppelin.dep;
import java.io.PrintStream;
import java.text.DecimalFormat;

62
zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java Normal file
View file

@ -0,0 +1,62 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.zeppelin.dep;
import static org.junit.Assert.assertTrue;
import java.io.File;
import org.apache.commons.io.FileUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
public class DependencyResolverTest {
private static DependencyResolver resolver;
private static String testPath;
private static String testCopyPath;
private static String home;
@BeforeClass
public static void setUp() throws Exception {
testPath = "test-repo";
testCopyPath = "test-copy-repo";
resolver = new DependencyResolver(testPath);
home = System.getenv("ZEPPELIN_HOME");
if (home == null) {
home = System.getProperty("zeppelin.home");
}
if (home == null) {
home = "..";
}
}
@AfterClass
public static void tearDown() throws Exception {
FileUtils.deleteDirectory(new File(home + "/" + testPath));
FileUtils.deleteDirectory(new File(home + "/" + testCopyPath));
}
@Test
public void testLoad() throws Exception {
resolver.load("org.apache.commons:commons-lang3:3.4", testCopyPath);
assertTrue(new File(home + "/" + testPath + "/org/apache/commons/commons-lang3/3.4/").exists());
assertTrue(new File(home + "/" + testCopyPath + "/commons-lang3-3.4.jar").exists());
}
}

5
zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
View file

@ -30,6 +30,7 @@ import javax.ws.rs.core.Application;
import org.apache.cxf.jaxrs.servlet.CXFNonSpringJaxrsServlet;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.notebook.Notebook;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
@ -73,12 +74,14 @@ public class ZeppelinServer extends Application {
private InterpreterFactory replFactory;
private NotebookRepo notebookRepo;
private SearchService notebookIndex;
private DependencyResolver depResolver;
public ZeppelinServer() throws Exception {
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
this.depResolver = new DependencyResolver(conf.getString(ConfVars.ZEPPELIN_DEP_LOCALREPO));
this.schedulerFactory = new SchedulerFactory();
this.replFactory = new InterpreterFactory(conf, notebookWsServer);
this.replFactory = new InterpreterFactory(conf, notebookWsServer, depResolver);
this.notebookRepo = new NotebookRepoSync(conf);
this.notebookIndex = new LuceneSearch();

2
zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js
View file

@ -960,7 +960,7 @@ angular.module('zeppelinWebApp')
clearUnknownColsFromGraphOption();
// set graph height
var height = $scope.paragraph.config.graph.height;
angular.element('#p' + $scope.paragraph.id + '_resize').height(height);
angular.element('#p' + $scope.paragraph.id + '_graph').height(height);
if (!type || type === 'table') {
setTable($scope.paragraph.result, refresh);

2
zeppelin-web/src/app/notebook/paragraph/paragraph.css
View file

@ -202,7 +202,6 @@
font-weight: bold;
font-family: 'Roboto', sans-serif;
font-size: 17px !important;
text-transform: capitalize;
}
.paragraph .title input {
@ -215,7 +214,6 @@
border-radius: 0;
-webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, .075);
box-shadow: inset 0 1px 1px rgba(0, 0, 0, .075);
text-transform: capitalize;
font-family: 'Roboto', sans-serif;
font-size: 17px !important;
font-weight: bold;

1
zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
View file

@ -429,6 +429,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
// Decide when new note is created, interpreter settings will be binded automatically or not.
ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING("zeppelin.notebook.autoInterpreterBinding", true),
ZEPPELIN_CONF_DIR("zeppelin.conf.dir", "conf"),
ZEPPELIN_DEP_LOCALREPO("zeppelin.dep.localrepo", "local-repo"),
// Allows a way to specify a ',' separated list of allowed origins for rest and websockets
// i.e. http://localhost:8080
ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"),

12
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
View file

@ -23,6 +23,7 @@ import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.NullArgumentException;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.dep.DependencyResolver;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.AngularObjectRegistryListener;
import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter;
@ -65,19 +66,24 @@ public class InterpreterFactory {
AngularObjectRegistryListener angularObjectRegistryListener;
DependencyResolver depResolver;
public InterpreterFactory(ZeppelinConfiguration conf,
AngularObjectRegistryListener angularObjectRegistryListener)
AngularObjectRegistryListener angularObjectRegistryListener,
DependencyResolver depResolver)
throws InterpreterException, IOException {
this(conf, new InterpreterOption(true), angularObjectRegistryListener);
this(conf, new InterpreterOption(true), angularObjectRegistryListener, depResolver);
}
public InterpreterFactory(ZeppelinConfiguration conf, InterpreterOption defaultOption,
AngularObjectRegistryListener angularObjectRegistryListener)
AngularObjectRegistryListener angularObjectRegistryListener,
DependencyResolver depResolver)
throws InterpreterException, IOException {
this.conf = conf;
this.defaultOption = defaultOption;
this.angularObjectRegistryListener = angularObjectRegistryListener;
this.depResolver = depResolver;
String replsConf = conf.getString(ConfVars.ZEPPELIN_INTERPRETERS);
interpreterClassList = replsConf.split(",");

62
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
View file

@ -38,32 +38,32 @@ import org.junit.Test;
public class InterpreterFactoryTest {
private InterpreterFactory factory;
private InterpreterFactory factory;
private File tmpDir;
private ZeppelinConfiguration conf;
private InterpreterContext context;
@Before
public void setUp() throws Exception {
public void setUp() throws Exception {
tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis());
tmpDir.mkdirs();
new File(tmpDir, "conf").mkdirs();
MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1");
MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2");
MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2");
System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
System.setProperty(ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1,org.apache.zeppelin.interpreter.mock.MockInterpreter2");
conf = new ZeppelinConfiguration();
factory = new InterpreterFactory(conf, new InterpreterOption(false), null);
context = new InterpreterContext("note", "id", "title", "text", null, null, null, null);
System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath());
System.setProperty(ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1,org.apache.zeppelin.interpreter.mock.MockInterpreter2");
conf = new ZeppelinConfiguration();
factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null);
context = new InterpreterContext("note", "id", "title", "text", null, null, null, null);
}
}
@After
public void tearDown() throws Exception {
delete(tmpDir);
}
@After
public void tearDown() throws Exception {
delete(tmpDir);
}
private void delete(File file){
if(file.isFile()) file.delete();
@ -78,24 +78,24 @@ public class InterpreterFactoryTest {
}
}
@Test
public void testBasic() {
List<String> all = factory.getDefaultInterpreterSettingList();
@Test
public void testBasic() {
List<String> all = factory.getDefaultInterpreterSettingList();
// get interpreter
Interpreter repl1 = factory.get(all.get(0)).getInterpreterGroup().getFirst();
assertFalse(((LazyOpenInterpreter) repl1).isOpen());
repl1.interpret("repl1", context);
assertTrue(((LazyOpenInterpreter) repl1).isOpen());
// get interpreter
Interpreter repl1 = factory.get(all.get(0)).getInterpreterGroup().getFirst();
assertFalse(((LazyOpenInterpreter) repl1).isOpen());
repl1.interpret("repl1", context);
assertTrue(((LazyOpenInterpreter) repl1).isOpen());
// try to get unavailable interpreter
assertNull(factory.get("unknown"));
// try to get unavailable interpreter
assertNull(factory.get("unknown"));
// restart interpreter
factory.restart(all.get(0));
repl1 = factory.get(all.get(0)).getInterpreterGroup().getFirst();
assertFalse(((LazyOpenInterpreter) repl1).isOpen());
}
// restart interpreter
factory.restart(all.get(0));
repl1 = factory.get(all.get(0)).getInterpreterGroup().getFirst();
assertFalse(((LazyOpenInterpreter) repl1).isOpen());
}
@Test
public void testFactoryDefaultList() throws IOException {
@ -119,8 +119,8 @@ public class InterpreterFactoryTest {
try {
factory.add("a mock", "mock2", null, new Properties());
} catch(NullArgumentException e) {
assertEquals("Test null option" , e.getMessage(),new NullArgumentException("option").getMessage());
}
assertEquals("Test null option" , e.getMessage(),new NullArgumentException("option").getMessage());
}
try {
factory.add("a mock" , "mock2" , new InterpreterOption(false),null);
} catch (NullArgumentException e){
@ -140,7 +140,7 @@ public class InterpreterFactoryTest {
factory.add("newsetting", "mock1", new InterpreterOption(false), new Properties());
assertEquals(3, factory.get().size());
InterpreterFactory factory2 = new InterpreterFactory(conf, null);
InterpreterFactory factory2 = new InterpreterFactory(conf, null, null);
assertEquals(3, factory2.get().size());
}
}

2
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java
View file

@ -58,7 +58,7 @@ public class NoteInterpreterLoaderTest {
MockInterpreter11.register("mock11", "group1", "org.apache.zeppelin.interpreter.mock.MockInterpreter11");
MockInterpreter2.register("mock2", "group2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2");
factory = new InterpreterFactory(conf, new InterpreterOption(false), null);
factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null);
}
@After

4
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java
View file

@ -85,7 +85,7 @@ public class NotebookTest implements JobListenerFactory{
MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1");
MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2");
factory = new InterpreterFactory(conf, new InterpreterOption(false), null);
factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null);
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);
@ -172,7 +172,7 @@ public class NotebookTest implements JobListenerFactory{
note.persist();
Notebook notebook2 = new Notebook(
conf, notebookRepo, schedulerFactory, new InterpreterFactory(conf, null), this, null);
conf, notebookRepo, schedulerFactory, new InterpreterFactory(conf, null, null), this, null);
assertEquals(1, notebook2.getAllNotes().size());
}

2
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
View file

@ -87,7 +87,7 @@ public class NotebookRepoSyncTest implements JobListenerFactory {
MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1");
MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2");
factory = new InterpreterFactory(conf, new InterpreterOption(false), null);
factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null);
SearchService search = mock(SearchService.class);
notebookRepoSync = new NotebookRepoSync(conf);

2
zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java
View file

@ -76,7 +76,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory {
MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1");
this.schedulerFactory = new SchedulerFactory();
factory = new InterpreterFactory(conf, new InterpreterOption(false), null);
factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null);
SearchService search = mock(SearchService.class);
notebookRepo = new VFSNotebookRepo(conf);