Elgato_dark/zeppelin
1
0
Fork 0
mirror of https://github.com/apache/zeppelin synced 2026-05-24 09:38:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

[ZEPPELIN-5525] Python vanillar interpreter doesn't' work in Python 3.8

Browse source
This commit is contained in:
Jeff Zhang 2021-09-16 16:41:46 +08:00
parent 056f952095
commit b816a530c0
12 changed files with 231 additions and 124 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
.github/workflows/core.yml vendored
View file

@ -32,6 +32,7 @@ jobs:
fail-fast: false
matrix:
hadoop: [hadoop2, hadoop3]
python: [3.7, 3.8]
steps:
- name: Checkout
uses: actions/checkout@v2
@ -51,12 +52,12 @@ jobs:
key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-zeppelin-
- name: Setup conda environment with python 3.7 and R
- name: Setup conda environment with python ${{ matrix.python }} and R
uses: conda-incubator/setup-miniconda@v2
with:
activate-environment: python_3_with_R
environment-file: testing/env_python_3_with_R.yml
python-version: 3.7
environment-file: testing/env_python_${{ matrix.python }}_with_R.yml
python-version: ${{ matrix.python }}
auto-activate-base: false
channel-priority: strict
- name: Make IRkernel available to Jupyter
@ -298,6 +299,10 @@ jobs:
spark-3-0-and-scala-2-12-and-other-interpreter:
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
python: [ 3.7, 3.8 ]
steps:
- name: Checkout
uses: actions/checkout@v2
@ -317,12 +322,12 @@ jobs:
key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-zeppelin-
- name: Setup conda environment with python 3.7 and R
- name: Setup conda environment with python ${{ matrix.python }} and R
uses: conda-incubator/setup-miniconda@v2
with:
activate-environment: python_3_with_R
environment-file: testing/env_python_3_with_R.yml
python-version: 3.7
environment-file: testing/env_python_${{ matrix.python }}_with_R.yml
python-version: ${{ matrix.python }}
auto-activate-base: false
- name: Make IRkernel available to Jupyter
run: |
@ -330,11 +335,15 @@ jobs:
- name: install environment
run: |
mvn install -DskipTests -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.0 -Pspark-scala-2.12 -Phadoop2 -B
- name: run tests
- name: run tests with ${{ matrix.python }}
run: mvn test -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.0 -Pspark-scala-2.12 -Phadoop2 -B -Dtest=org.apache.zeppelin.spark.*,apache.zeppelin.python.*,apache.zeppelin.jupyter.*,apache.zeppelin.r.* -DfailIfNoTests=false
spark-3-1-and-scala-2-12-and-other-interpreter:
runs-on: ubuntu-20.04
strategy:
fail-fast: false
matrix:
python: [ 3.7, 3.8 ]
steps:
- name: Checkout
uses: actions/checkout@v2
@ -354,19 +363,19 @@ jobs:
key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-zeppelin-
- name: Setup conda environment with python 3.7 and R
- name: Setup conda environment with python ${{ matrix.python }} and R
uses: conda-incubator/setup-miniconda@v2
with:
activate-environment: python_3_with_R
environment-file: testing/env_python_3_with_R.yml
python-version: 3.7
environment-file: testing/env_python_${{ matrix.python }}_with_R.yml
python-version: ${{ matrix.python }}
auto-activate-base: false
- name: Make IRkernel available to Jupyter
run: |
R -e "IRkernel::installspec()"
- name: install environment
run: mvn install -DskipTests -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.1 -Pspark-scala-2.12 -Phadoop2 -B
- name: run tests
- name: run tests with ${{ matrix.python }}
run: mvn test -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.1 -Pspark-scala-2.12 -Phadoop2 -B -Dtest=org.apache.zeppelin.spark.*,apache.zeppelin.python.*,apache.zeppelin.jupyter.*,apache.zeppelin.r.* -DfailIfNoTests=false
test-livy-0-5-with-spark-2-2-0-under-python3:
runs-on: ubuntu-20.04

6
flink/flink-scala-parent/src/test/java/org/apache/zeppelin/flink/IPyFlinkInterpreterTest.java
View file

@ -62,6 +62,12 @@ public class IPyFlinkInterpreterTest extends IPythonInterpreterTest {
private LazyOpenInterpreter flinkScalaInterpreter;
public IPyFlinkInterpreterTest() {
super();
// disable bokeh test, because its (bokeh2) dependencies conflicts with apache-flink,
this.enableBokehTest = false;
}
protected Properties initIntpProperties() {
Properties p = new Properties();
p.setProperty("zeppelin.pyflink.python", "python");

13
python/src/main/resources/python/zeppelin_python.py
View file

@ -77,6 +77,15 @@ class PythonCompletion:
result = json.dumps(list(filter(lambda x : not re.match("^__.*", x), list(completionList))))
self.interpreter.setStatementsFinished(result, False)
# ast api is changed after python 3.8, see https://github.com/ipython/ipython/pull/11593
if sys.version_info > (3,8):
from ast import Module
else :
# mock the new API, ignore second argument
# see https://github.com/ipython/ipython/issues/11590
from ast import Module as OriginalModule
Module = lambda nodelist, type_ignores: OriginalModule(nodelist)
host = sys.argv[1]
port = int(sys.argv[2])
@ -148,7 +157,7 @@ while True :
[code.body[-(nhooks + 1)]] if len(code.body) > nhooks else [])
try:
for node in to_run_exec:
mod = ast.Module([node])
mod = Module([node], [])
code = compile(mod, '<stdin>', 'exec')
exec(code, _zcUserQueryNameSpace)
@ -158,7 +167,7 @@ while True :
exec(code, _zcUserQueryNameSpace)
for node in to_run_hooks:
mod = ast.Module([node])
mod = Module([node], [])
code = compile(mod, '<stdin>', 'exec')
exec(code, _zcUserQueryNameSpace)

24
python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java
View file

@ -29,6 +29,8 @@ import org.apache.zeppelin.interpreter.InterpreterResultMessage;
import org.apache.zeppelin.interpreter.LazyOpenInterpreter;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
@ -49,6 +51,10 @@ import static org.junit.Assert.fail;
public class IPythonInterpreterTest extends BasePythonInterpreterTest {
private static final Logger LOGGER = LoggerFactory.getLogger(IPythonInterpreterTest.class);
protected boolean enableBokehTest = true;
protected Properties initIntpProperties() {
Properties properties = new Properties();
properties.setProperty("zeppelin.python.maxResult", "3");
@ -215,14 +221,18 @@ public class IPythonInterpreterTest extends BasePythonInterpreterTest {
assertTrue("No Image Output", hasImageOutput);
assertTrue("No Line Text", hasLineText);
if (!enableBokehTest) {
LOGGER.info("Bokeh test is skipped");
return;
}
// bokeh
// bokeh initialization
context = getInterpreterContext();
result = interpreter.interpret("from bokeh.io import output_notebook, show\n" +
"from bokeh.plotting import figure\n" +
"import bkzep\n" +
"output_notebook(notebook_type='zeppelin')", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
"output_notebook()", context);
assertEquals(context.out.toString(), InterpreterResult.Code.SUCCESS, result.code());
interpreterResultMessages = context.out.toInterpreterResultMessage();
if (interpreterResultMessages.size() == 3) {
@ -299,14 +309,14 @@ public class IPythonInterpreterTest extends BasePythonInterpreterTest {
InterpreterResult.Code.SUCCESS, result.code());
interpreterResultMessages = context.out.toInterpreterResultMessage();
assertEquals(context.out.toString(), 5, interpreterResultMessages.size());
assertEquals(interpreterResultMessages.size() + ":" + context.out.toString(),
3, interpreterResultMessages.size());
// the first message is the warning text message.
assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(0).getType());
assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(1).getType());
assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(2).getType());
assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(3).getType());
assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(4).getType());
// docs_json is the source data of plotting which bokeh would use to render the plotting.
assertTrue(interpreterResultMessages.get(4).getData().contains("docs_json"));
assertTrue(interpreterResultMessages.get(2).getData().contains("docs_json"));
}

33
testing/env_python_3.7_with_R.yml Normal file
View file

@ -0,0 +1,33 @@
name: python_3_with_R
channels:
- conda-forge
- defaults
dependencies:
- pycodestyle
- scipy
- numpy=1.19.5
- grpcio
- protobuf
- pandasql
- ipython
- ipykernel
- jupyter_client=5
- hvplot
- plotnine
- seaborn
- intake
- intake-parquet
- intake-xarray
- altair
- vega_datasets
- plotly
- pip
- r-base=3
- r-data.table
- r-evaluate
- r-base64enc
- r-knitr
- r-ggplot2
- r-irkernel
- r-shiny
- r-googlevis

33
testing/env_python_3.8_with_R.yml Normal file
View file

@ -0,0 +1,33 @@
name: python_3_with_R
channels:
- conda-forge
- defaults
dependencies:
- pycodestyle
- scipy
- numpy=1.19.5
- grpcio
- protobuf
- pandasql
- ipython
- ipykernel
- jupyter_client=5
- hvplot
- plotnine
- seaborn
- intake
- intake-parquet
- intake-xarray
- altair
- vega_datasets
- plotly
- pip
- r-base=3
- r-data.table
- r-evaluate
- r-base64enc
- r-knitr
- r-ggplot2
- r-irkernel
- r-shiny
- r-googlevis

37
testing/env_python_3_with_R.yml
View file

@ -4,30 +4,31 @@ channels:
- defaults
dependencies:
- pycodestyle
- numpy=1
- pandas=0.25
- scipy=1
- grpcio=1.22.0
- hvplot=0.5.2
- protobuf=3
- pandasql=0.7.3
- ipython=7
- matplotlib=3
- ipykernel=5
- scipy
- numpy=1.19.5
- grpcio
- protobuf
- pandasql
- ipython
- ipykernel
- jupyter_client=5
- bokeh=1.3.4
- panel=0.6.0
- holoviews=1.12.3
- pyyaml=3
- hvplot
- plotnine
- seaborn
- bokeh
- intake
- intake-parquet
- intake-xarray
- altair
- vega_datasets
- plotly
- pip
- pip:
- bkzep==0.6.1
- r-base=3
- r-data.table
- r-evaluate
- r-base64enc
- r-knitr
- r-ggplot2
- r-irkernel
- r-shiny
- r-googlevis
- r-googlevis

36
testing/env_python_3_with_R_and_tensorflow.yml
View file

@ -4,26 +4,27 @@ channels:
- defaults
dependencies:
- pycodestyle
- numpy=1
- pandas=0.25
- scipy=1
- grpcio=1.22.0
- hvplot=0.5.2
- protobuf=3
- pandasql=0.7.3
- ipython=7
- matplotlib=3
- ipykernel=5
- scipy
- numpy=1.19.5
- grpcio
- protobuf
- pandasql
- ipython
- ipykernel
- jupyter_client=5
- bokeh=1.3.4
- panel=0.6.0
- holoviews=1.12.3
- pyyaml=3
- hvplot
- plotnine
- seaborn
- bokeh
- intake
- intake-parquet
- intake-xarray
- altair
- vega_datasets
- plotly
- pip
- pip:
- bkzep==0.6.1
- r-base=3
- r-data.table
- r-evaluate
- r-base64enc
- r-knitr
@ -31,5 +32,4 @@ dependencies:
- r-irkernel
- r-shiny
- r-googlevis
- tensorflow=1.13

34
testing/env_python_3_with_flink_110.yml
View file

@ -3,22 +3,24 @@ channels:
- conda-forge
- defaults
dependencies:
- pycodestyle
- scipy
- numpy=1.19.5
- grpcio
- protobuf
- pandasql
- ipython
- ipykernel
- jupyter_client=5
- hvplot
- plotnine
- seaborn
- intake
- intake-parquet
- intake-xarray
- altair
- vega_datasets
- plotly
- pip
- pip:
- apache-flink==1.10.2
- bkzep==0.6.1
- numpy==1.17.3
- pandas==0.25.0
- scipy==1.3.1
- grpcio==1.34.1
- hvplot==0.5.2
- protobuf==3.10.0
- pandasql==0.7.3
- ipython==7.8.0
- matplotlib==3.0.3
- ipykernel==5.1.2
- jupyter_client==5.3.4
- bokeh==1.3.4
- panel==0.6.0
- holoviews==1.12.3
- pycodestyle==2.5.0

34
testing/env_python_3_with_flink_111.yml
View file

@ -4,22 +4,24 @@ channels:
- defaults
dependencies:
- pycodestyle
- numpy=1
- pandas=0.25
- scipy=1
- grpcio=1.22.0
- hvplot=0.5.2
- protobuf=3
- pandasql=0.7.3
- ipython=7
- matplotlib=3
- ipykernel=5
- scipy
- numpy=1.19.5
- grpcio
- protobuf
- pandasql
- ipython
- ipykernel
- jupyter_client=5
- bokeh=1.3.4
- panel=0.6.0
- holoviews=1.12.3
- pyyaml=3
- hvplot
- plotnine
- seaborn
- intake
- intake-parquet
- intake-xarray
- altair
- vega_datasets
- plotly
- pip
- pip:
- bkzep==0.6.1
- apache-flink==1.11.1
- apache-flink==1.11.3

37
testing/env_python_3_with_flink_112.yml
View file

@ -3,24 +3,25 @@ channels:
- conda-forge
- defaults
dependencies:
- pycodestyle
- scipy
- numpy=1.19.5
- grpcio
- protobuf
- pandasql
- ipython
- ipykernel
- jupyter_client=5
- hvplot
- plotnine
- seaborn
- intake
- intake-parquet
- intake-xarray
- altair
- vega_datasets
- plotly
- pip
- pip:
- bkzep==0.6.1
- apache-flink==1.12.0
- pycodestyle
- numpy=1
- pandas=0.25
- scipy=1
- grpcio=1.22.0
- hvplot=0.5.2
- protobuf=3
- pandasql=0.7.3
- ipython=7
- matplotlib=3
- ipykernel=5
- jupyter_client=5
- bokeh=1.3.4
- panel=0.6.0
- holoviews=1.12.3
- pyyaml=3
- apache-flink==1.12.2

37
testing/env_python_3_with_flink_113.yml
View file

@ -3,24 +3,25 @@ channels:
- conda-forge
- defaults
dependencies:
- pycodestyle
- scipy
- numpy=1.19.5
- grpcio
- protobuf
- pandasql
- ipython
- ipykernel
- jupyter_client=5
- hvplot
- plotnine
- seaborn
- intake
- intake-parquet
- intake-xarray
- altair
- vega_datasets
- plotly
- pip
- pip:
- bkzep==0.6.1
- apache-flink==1.13.0
- pycodestyle
- numpy=1
- pandas=0.25
- scipy=1
- grpcio=1.22.0
- hvplot=0.5.2
- protobuf=3
- pandasql=0.7.3
- ipython=7
- matplotlib=3
- ipykernel=5
- jupyter_client=5
- bokeh=1.3.4
- panel=0.6.0
- holoviews=1.12.3
- pyyaml=3
- apache-flink==1.13.1