Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

[ZEPPELIN-3092] Add remote Github repository synchronzing

Browse source 
- Pull the latest changes when the notebook page loads in the browser
- After commiting the changes, pull the latest changes from the remote repo then push the latest changes to the remote repo
This commit is contained in:
Mohamed Magdy 2017-12-05 17:54:42 +01:00
parent 32f6764b19
commit 33ae24a473
6 changed files with 111 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
View file

@ -546,8 +546,14 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getString(ConfVars.ZEPPELIN_INTERPRETER_LIFECYCLE_MANAGER_CLASS);
}
public String getZeppelinNotebookGitURL() { return getString(ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_URL); }
public String getZeppelinNotebookGitUsername() { return getString(ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_USERNAME); }
public String getZeppelinNotebookGitURL() {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_URL);
}
public String getZeppelinNotebookGitUsername() {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_USERNAME);
}
public String getZeppelinNotebookGitAccessToken() {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_ACCESS_TOKEN);
}

12
zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
View file

@ -209,18 +209,6 @@ public class ZeppelinServer extends Application {
}
});
// when zeppelin is started inside of ide (especially for eclipse)
// for graceful shutdown, input any key in console window
if (System.getenv("ZEPPELIN_IDENT_STRING") == null) {
try {
System.in.read();
} catch (IOException e) {
LOG.error("Exception in ZeppelinServer while main ", e);
}
System.exit(0);
}
jettyWebServer.join();
ZeppelinServer.notebook.getInterpreterSettingManager().close();
}

4
zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java
View file

@ -823,9 +823,9 @@ public class NotebookServer extends WebSocketServlet
String user = fromMessage.principal;
Note note = notebook.getNote(noteId);
if (note != null) {
Note note = notebook.getNoteAfterReloadFromRepo(noteId);
if (note != null) {
if (!hasParagraphReaderPermission(conn, notebook, noteId,
userAndRoles, fromMessage.principal, "read")) {
return;

5
zeppelin-zengine/pom.xml
View file

@ -625,6 +625,11 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
<version>4.3.1.201605051710-r</version>
</dependency>
</dependencies>
<build>

6
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
View file

@ -34,6 +34,7 @@ import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.Sets;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.interpreter.*;
import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.quartz.CronScheduleBuilder;
@ -295,6 +296,11 @@ public class Notebook implements NoteEventListener {
}
}
public Note getNoteAfterReloadFromRepo(String noteId) {
loadNoteFromRepo(noteId, new AuthenticationInfo(StringUtils.EMPTY));
return getNote(noteId);
}
public Folder getFolder(String folderId) {
synchronized (folders) {
return folders.getFolder(folderId);

93
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java
View file

@ -19,13 +19,14 @@ package org.apache.zeppelin.notebook.repo;
import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.Collection;
import java.util.List;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.*;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.NoHeadException;
import org.eclipse.jgit.diff.DiffEntry;
@ -35,6 +36,7 @@ import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.transport.*;
import org.eclipse.jgit.treewalk.filter.PathFilter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -56,17 +58,25 @@ public class GitNotebookRepo extends VFSNotebookRepo {
private String localPath;
private Git git;
private ZeppelinConfiguration zeppelinConfiguration;
public GitNotebookRepo(ZeppelinConfiguration conf) throws IOException {
super(conf);
this.zeppelinConfiguration = conf;
localPath = getRootDir().getName().getPath();
LOG.info("Opening a git repo at '{}'", localPath);
LOG.debug("Opening a git repo at '{}'", localPath);
Repository localRepo = new FileRepository(Joiner.on(File.separator).join(localPath, ".git"));
if (!localRepo.getDirectory().exists()) {
LOG.info("Git repo {} does not exist, creating a new one", localRepo.getDirectory());
LOG.debug("Git repo {} does not exist, creating a new one", localRepo.getDirectory());
localRepo.create();
}
git = new Git(localRepo);
configureRemoteStream();
pullFromRemoteStream();
}
@Override
@ -91,6 +101,10 @@ public class GitNotebookRepo extends VFSNotebookRepo {
LOG.debug("{} changes are about to be commited", added.getEntryCount());
RevCommit commit = git.commit().setMessage(commitMessage).call();
revision = new Revision(commit.getName(), commit.getShortMessage(), commit.getCommitTime());
if(isRemoteStreamUpdatable())
updateRemoteStream();
} else {
LOG.debug("No changes found {}", pattern);
}
@ -147,6 +161,8 @@ public class GitNotebookRepo extends VFSNotebookRepo {
List<Revision> history = Lists.newArrayList();
LOG.debug("Listing history for {}:", noteId);
try {
pullFromRemoteStream();
Iterable<RevCommit> logs = git.log().addPath(noteId).call();
for (RevCommit log: logs) {
history.add(new Revision(log.getName(), log.getShortMessage(), log.getCommitTime()));
@ -164,6 +180,7 @@ public class GitNotebookRepo extends VFSNotebookRepo {
@Override
public Note setNoteRevision(String noteId, String revId, AuthenticationInfo subject)
throws IOException {
LOG.debug("Setting log revision");
Note revisionNote = get(noteId, revId, subject);
if (revisionNote != null) {
save(revisionNote, subject);
@ -185,4 +202,74 @@ public class GitNotebookRepo extends VFSNotebookRepo {
this.git = git;
}
private boolean isRemoteStreamUpdatable() {
return !zeppelinConfiguration.getZeppelinNotebookGitURL().isEmpty();
}
private void configureRemoteStream() {
if(!isRemoteStreamUpdatable())
return;
try {
LOG.debug("Setting up remote stream");
RemoteAddCommand remoteAddCommand = git.remoteAdd();
remoteAddCommand.setName("origin");
remoteAddCommand.setUri(new URIish(zeppelinConfiguration.getZeppelinNotebookGitURL()));
remoteAddCommand.call();
} catch (GitAPIException e) {
e.printStackTrace();
} catch (URISyntaxException e) {
e.printStackTrace();
}
}
private void updateRemoteStream() {
LOG.debug("Updating remote stream");
pullFromRemoteStream();
pushToRemoteSteam();
}
private void pullFromRemoteStream() {
if(!isRemoteStreamUpdatable())
return;
try {
LOG.debug("Pull latest changed from remote stream");
PullCommand pullCommand = git.pull();
pullCommand.setCredentialsProvider(
new UsernamePasswordCredentialsProvider(
zeppelinConfiguration.getZeppelinNotebookGitUsername(),
zeppelinConfiguration.getZeppelinNotebookGitAccessToken()
)
);
pullCommand.call();
} catch (GitAPIException e) {
LOG.error("Error when pulling latest changes from remote repository");
e.printStackTrace();
}
}
private void pushToRemoteSteam() {
if(!isRemoteStreamUpdatable())
return;
try {
LOG.debug("Push latest changed from remote stream");
PushCommand pushCommand = git.push();
pushCommand.setCredentialsProvider(
new UsernamePasswordCredentialsProvider(
zeppelinConfiguration.getZeppelinNotebookGitUsername(),
zeppelinConfiguration.getZeppelinNotebookGitAccessToken()
)
);
pushCommand.call();
} catch (GitAPIException e) {
LOG.error("Error when pushing latest changes from remote repository");
e.printStackTrace();
}
}
}