Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

ZEPPELIN-501: update notebooks on save + tests

Browse source
This commit is contained in:
Alexander Bezzubov 2015-12-22 18:22:59 +09:00
parent 3f209043a3
commit e915a694aa
4 changed files with 135 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
View file

@ -38,6 +38,7 @@ import org.apache.zeppelin.notebook.utility.IdHashes;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.scheduler.JobListener;
import org.apache.zeppelin.search.SearchService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -60,6 +61,7 @@ public class Note implements Serializable, JobListener {
private transient NoteInterpreterLoader replLoader;
private transient JobListenerFactory jobListenerFactory;
private transient NotebookRepo repo;
private transient SearchService index;
/**
* note configurations.
@ -78,10 +80,12 @@ public class Note implements Serializable, JobListener {
public Note() {}
public Note(NotebookRepo repo, NoteInterpreterLoader replLoader, JobListenerFactory jlFactory) {
public Note(NotebookRepo repo, NoteInterpreterLoader replLoader,
JobListenerFactory jlFactory, SearchService noteIndex) {
this.repo = repo;
this.replLoader = replLoader;
this.jobListenerFactory = jlFactory;
this.index = noteIndex;
generateId();
}
@ -293,7 +297,7 @@ public class Note implements Serializable, JobListener {
return paragraphs.get(paragraphs.size() - 1);
}
}
public List<Map<String, String>> generateParagraphsInfo (){
List<Map<String, String>> paragraphsInfo = new LinkedList<>();
synchronized (paragraphs) {
@ -307,7 +311,7 @@ public class Note implements Serializable, JobListener {
}
}
return paragraphsInfo;
}
}
/**
* Run all paragraphs sequentially.
@ -373,6 +377,7 @@ public class Note implements Serializable, JobListener {
public void persist() throws IOException {
snapshotAngularObjectRegistry();
index.updateIndexDoc(this);
repo.save(this);
}

2
zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
View file

@ -136,7 +136,7 @@ public class Notebook {
*/
public Note createNote(List<String> interpreterIds) throws IOException {
NoteInterpreterLoader intpLoader = new NoteInterpreterLoader(replFactory);
Note note = new Note(notebookRepo, intpLoader, jobListenerFactory);
Note note = new Note(notebookRepo, intpLoader, jobListenerFactory, notebookIndex);
intpLoader.setNoteId(note.id());
synchronized (notes) {
notes.put(note.id(), note);

104
zeppelin-zengine/src/main/java/org/apache/zeppelin/search/SearchService.java
View file

@ -160,19 +160,42 @@ public class SearchService {
}
} catch (IOException | InvalidTokenOffsetsException e) {
LOG.error("Exception on searching for {}", query, e);
;
}
return matchingParagraphs;
}
/**
* Updates all documents in index for the given note:
* - name
* - all paragraphs
*
* @param note a Note to update index for
* @throws IOException
*/
public void updateIndexDoc(Note note) throws IOException {
updateIndexNoteName(note);
for (Paragraph p: note.getParagraphs()) {
updateIndexParagraph(note, p);
}
}
private void updateIndexNoteName(Note note) throws IOException {
updateDoc(note.getId(), note.getName(), null);
}
void updateIndexDoc(Note note, Paragraph p) throws IOException {
private void updateIndexParagraph(Note note, Paragraph p) throws IOException {
updateDoc(note.getId(), note.getName(), p);
}
/**
* Updates index for the given note: either note.name or a paragraph
* If paragraph is <code>null</code> - updates only for the note.name
*
* @param noteId
* @param noteName
* @param p
* @throws IOException
*/
private void updateDoc(String noteId, String noteName, Paragraph p) throws IOException {
String id = formatId(noteId, p);
Document doc = newDocument(id, noteName, p);
@ -184,6 +207,44 @@ public class SearchService {
}
}
/**
* If paragraph is not null, id is <noteId>/paragraphs/<paragraphId>,
* otherwise it's just <noteId>.
*/
static String formatId(String noteId, Paragraph p) {
String id = noteId;
if (null != p) {
id = Joiner.on('/').join(id, "paragraphs", p.getId());
}
return id;
}
/**
* If paragraph is not null, indexes code in the paragraph,
* otherwise indexes the notebook name.
*
* @param id id of the document, different for Note name and paragraph
* @param noteName name of the note
* @param p paragraph
* @return
*/
private Document newDocument(String id, String noteName, Paragraph p) {
Document doc = new Document();
Field pathField = new StringField(ID_FIELD, id, Field.Store.YES);
doc.add(pathField);
doc.add(new StringField("title", noteName, Field.Store.YES));
if (null != p) {
doc.add(new TextField(SEARCH_FIELD, p.getText(), Field.Store.YES));
Date date = p.getDateStarted() != null ? p.getDateStarted() : p.getDateCreated();
doc.add(new LongField("modified", date.getTime(), Field.Store.NO));
} else {
doc.add(new TextField(SEARCH_FIELD, noteName, Field.Store.YES));
}
return doc;
}
/**
* Indexes full collection of notes: all the paragraphs + Note names
*
@ -277,43 +338,4 @@ public class SearchService {
w.addDocument(doc);
}
/**
* If paragraph is not null, indexes code in the paragraph,
* otherwise indexes the notebook name.
*
* @param id id of the document, different for Note name and paragraph
* @param noteName name of the note
* @param p paragraph
* @return
*/
private Document newDocument(String id, String noteName, Paragraph p) {
Document doc = new Document();
Field pathField = new StringField(ID_FIELD, id, Field.Store.YES);
doc.add(pathField);
doc.add(new StringField("title", noteName, Field.Store.YES));
if (null != p) {
doc.add(new TextField(SEARCH_FIELD, p.getText(), Field.Store.YES));
Date date = p.getDateStarted() != null ? p.getDateStarted() : p.getDateCreated();
doc.add(new LongField("modified", date.getTime(), Field.Store.NO));
} else {
doc.add(new TextField(SEARCH_FIELD, noteName, Field.Store.YES));
}
return doc;
}
/**
* If paragraph is not null, id is <noteId>/paragraphs/<paragraphId>,
* otherwise it's just <noteId>.
*/
static String formatId(String noteId, Paragraph p) {
String id = noteId;
if (null != p) {
id = Joiner.on('/').join(id, "paragraphs", p.getId());
}
return id;
}
}

66
zeppelin-zengine/src/test/java/org/apache/zeppelin/search/SearchServiceTest.java
View file

@ -17,6 +17,7 @@
package org.apache.zeppelin.search;
import static com.google.common.truth.Truth.assertThat;
import static org.mockito.Mockito.*;
import static org.apache.zeppelin.search.SearchService.formatId;
import java.io.IOException;
@ -24,15 +25,32 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.NoteInterpreterLoader;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import com.google.common.collect.ImmutableList;
public class SearchServiceTest {
SearchService notebookIndex;
private static NoteInterpreterLoader replLoaderMock;
private static NotebookRepo notebookRepoMock;
private SearchService notebookIndex;
@BeforeClass
public static void beforeStartUp() {
notebookRepoMock = mock(NotebookRepo.class);
replLoaderMock = mock(NoteInterpreterLoader.class);
when(replLoaderMock.getInterpreterSettings())
.thenReturn(ImmutableList.<InterpreterSetting>of());
}
@Before
public void startUp() {
@ -106,7 +124,7 @@ public class SearchServiceTest {
//when
Paragraph p2 = note2.getLastParagraph();
p2.setText("test indeed");
notebookIndex.updateIndexDoc(note2, p2);
notebookIndex.updateIndexDoc(note2);
//then
List<Map<String, String>> results = notebookIndex.query("all");
@ -135,6 +153,48 @@ public class SearchServiceTest {
assertThat(results.size()).isEqualTo(1);
}
@Test public void indexParagraphUpdatedOnNoteSave() throws IOException {
//given: total 2 notebooks, 3 paragraphs
Note note1 = newNoteWithParapgraph("Notebook1", "test");
Note note2 = newNoteWithParapgraphs("Notebook2", "not test", "not test at all");
notebookIndex.addIndexDocs(Arrays.asList(note1, note2));
assertThat(resultForQuery("test").size()).isEqualTo(3);
//when
Paragraph p1 = note1.getLastParagraph();
p1.setText("no no no");
note1.persist();
//then
assertThat(resultForQuery("Notebook1").size()).isEqualTo(1);
List<Map<String, String>> results = resultForQuery("test");
assertThat(results).isNotEmpty();
assertThat(results.size()).isEqualTo(2);
//does not include Notebook1's paragraph any more
for (Map<String, String> result: results) {
assertThat(result.get("id").startsWith(note1.getId())).isFalse();;
}
}
@Test public void indexNoteNameUpdatedOnNoteSave() throws IOException {
//given: total 2 notebooks, 3 paragraphs
Note note1 = newNoteWithParapgraph("Notebook1", "test");
Note note2 = newNoteWithParapgraphs("Notebook2", "not test", "not test at all");
notebookIndex.addIndexDocs(Arrays.asList(note1, note2));
assertThat(resultForQuery("test").size()).isEqualTo(3);
//when
note1.setName("NotebookN");
note1.persist();
//then
assertThat(resultForQuery("Notebook1")).isEmpty();
assertThat(resultForQuery("NotebookN")).isNotEmpty();
assertThat(resultForQuery("NotebookN").size()).isEqualTo(1);
}
private List<Map<String, String>> resultForQuery(String q) {
return notebookIndex.query(q);
}
@ -172,7 +232,7 @@ public class SearchServiceTest {
}
private Note newNote(String name) {
Note note = new Note(null, null, null);
Note note = new Note(notebookRepoMock, replLoaderMock, null, notebookIndex);
note.setName(name);
return note;
}