Unable to init spark shell in wangler with spark 2.3.2

Description

I setup kylo 0.10 on ali emr which based on hadoop 2.7.3, spark2.3.2. data ingest is fine but no luck in the wangler:

2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - 2019-01-31 10:15:59 INFO main:RemoteClientRunner:119 - Registering client bb07416b-500a-4df0-b9fb-dad6cde06217 at emr-header-1.cluster-92503:45000 with server http://emr-header-1.cluster-92503:8400/proxy/v1/spark/shell/register.
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - 2019-01-31 10:15:59 ERROR main:RemoteClientRunner:139 - Failed to register Spark Shell client: java.lang.ClassCastException: com.alibaba.fastjson.support.jaxrs.FastJsonAutoDiscoverable cannot be cast to org.glassfish.jersey.internal.spi.AutoDiscoverable
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - java.lang.ClassCastException: com.alibaba.fastjson.support.jaxrs.FastJsonAutoDiscoverable cannot be cast to org.glassfish.jersey.internal.spi.AutoDiscoverable
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.glassfish.jersey.model.internal.CommonConfig$2.compare(CommonConfig.java:594)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at java.util.TreeMap.put(TreeMap.java:552)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at java.util.TreeSet.add(TreeSet.java:255)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at java.util.AbstractCollection.addAll(AbstractCollection.java:344)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at java.util.TreeSet.addAll(TreeSet.java:312)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.glassfish.jersey.model.internal.CommonConfig.configureAutoDiscoverableProviders(CommonConfig.java:616)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.glassfish.jersey.client.ClientConfig$State.configureAutoDiscoverableProviders(ClientConfig.java:364)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.glassfish.jersey.client.ClientConfig$State.initRuntime(ClientConfig.java:399)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.glassfish.jersey.client.ClientConfig$State.access$000(ClientConfig.java:90)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.glassfish.jersey.client.ClientConfig$State$3.get(ClientConfig.java:122)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.glassfish.jersey.client.ClientConfig$State$3.get(ClientConfig.java:119)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.glassfish.jersey.internal.util.collection.Values$LazyValueImpl.get(Values.java:340)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.glassfish.jersey.client.ClientConfig.getRuntime(ClientConfig.java:733)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.glassfish.jersey.client.ClientRequest.getConfiguration(ClientRequest.java:286)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.glassfish.jersey.client.JerseyInvocation.validateHttpMethodAndEntity(JerseyInvocation.java:135)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.glassfish.jersey.client.JerseyInvocation.<init>(JerseyInvocation.java:105)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.glassfish.jersey.client.JerseyInvocation.<init>(JerseyInvocation.java:101)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.glassfish.jersey.client.JerseyInvocation.<init>(JerseyInvocation.java:92)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.glassfish.jersey.client.JerseyInvocation$Builder.method(JerseyInvocation.java:437)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.glassfish.jersey.client.JerseyInvocation$Builder.post(JerseyInvocation.java:343)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at com.thinkbiganalytics.rest.JerseyRestClient.post(JerseyRestClient.java:576)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at com.thinkbiganalytics.spark.RemoteClientRunner.register(RemoteClientRunner.java:120)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at com.thinkbiganalytics.spark.RemoteClientRunner.run(RemoteClientRunner.java:136)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.springframework.boot.SpringApplication.callRunner(SpringApplication.java:723)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.springframework.boot.SpringApplication.callRunners(SpringApplication.java:713)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.springframework.boot.SpringApplication.afterRefresh(SpringApplication.java:703)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.springframework.boot.SpringApplication.run(SpringApplication.java:304)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at com.thinkbiganalytics.spark.SparkShellApp.main(SparkShellApp.java:88)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at java.lang.reflect.Method.invoke(Method.java:498)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:896)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:198)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:228)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:137)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
2019-01-31 10:15:59 INFO launcher-proc-1:SparkShellApp:61 - 2019-01-31 10:15:59 INFO Thread-1:SparkContext:54 - Invoking stop() from shutdown hook

I have checked that

https://github.com/alibaba/fastjson/blob/master/src/main/java/com/alibaba/fastjson/support/jaxrs/FastJsonAutoDiscoverable.java

already implemented

https://github.com/jersey/jersey/blob/2.23.2/core-common/src/main/java/org/glassfish/jersey/internal/spi/AutoDiscoverable.java

I had tried to delete the fastjson-1.2.45.jar, no luck with error:

2019-02-14 14:57:40 INFO launcher-proc-1:SparkShellApp:61 - 2019-02-14 14:57:40 INFO main:RemoteClientRunner:125 - Registra
tion failed with response: InboundJaxrsResponse{context=ClientResponse{method=POST, uri=http://emr-header-1.cluster-92503:840
0/proxy/v1/spark/shell/register, status=400, reason=Bad Request}}
2019-02-14 14:57:40 INFO launcher-proc-1:SparkShellApp:61 - 2019-02-14 14:57:40 ERROR main:RemoteClientRunner:139 - Failed t
o register Spark Shell client: java.lang.IllegalStateException: Failed to register with server
2019-02-14 14:57:40 INFO launcher-proc-1:SparkShellApp:61 - java.lang.IllegalStateException: Failed to register with server
2019-02-14 14:57:40 INFO launcher-proc-1:SparkShellApp:61 - at com.thinkbiganalytics.spark.RemoteClientRunner.register(Re
moteClientRunner.java:126)
2019-02-14 14:57:40 INFO launcher-proc-1:SparkShellApp:61 - at com.thinkbiganalytics.spark.RemoteClientRunner.run(RemoteC
lientRunner.java:136)
2019-02-14 14:57:40 INFO launcher-proc-1:SparkShellApp:61 - at org.springframework.boot.SpringApplication.callRunner(Spri
ngApplication.java:723)
2019-02-14 14:57:40 INFO launcher-proc-1:SparkShellApp:61 - at org.springframework.boot.SpringApplication.callRunners(Spr
ingApplication.java:713)
2019-02-14 14:57:40 INFO launcher-proc-1:SparkShellApp:61 - at org.springframework.boot.SpringApplication.afterRefresh(Sp
ringApplication.java:703)
2019-02-14 14:57:40 INFO launcher-proc-1:SparkShellApp:61 - at org.springframework.boot.SpringApplication.run(SpringAppli
cation.java:304)
2019-02-14 14:57:40 INFO launcher-proc-1:SparkShellApp:61 - at com.thinkbiganalytics.spark.SparkShellApp.main(SparkShellA
pp.java:88)
2019-02-14 14:57:40 INFO launcher-proc-1:SparkShellApp:61 - at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method
)
2019-02-14 14:57:40 INFO launcher-proc-1:SparkShellApp:61 - at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAc
cessorImpl.java:62)
2019-02-14 14:57:40 INFO launcher-proc-1:SparkShellApp:61 - at sun.reflect.DelegatingMethodAccessorImpl.invoke(Delegating
MethodAccessorImpl.java:43)
2019-02-14 14:57:40 INFO launcher-proc-1:SparkShellApp:61 - at java.lang.reflect.Method.invoke(Method.java:498)
2019-02-14 14:57:40 INFO launcher-proc-1:SparkShellApp:61 - at org.apache.spark.deploy.JavaMainApplication.start(SparkApp
lication.scala:52)
2019-02-14 14:57:40 INFO launcher-proc-1:SparkShellApp:61 - at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$depl
oy$SparkSubmit$$runMain(SparkSubmit.scala:896)
2019-02-14 14:57:40 INFO launcher-proc-1:SparkShellApp:61 - at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubm
it.scala:198)
2019-02-14 14:57:40 INFO launcher-proc-1:SparkShellApp:61 - at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.sc
ala:228)
2019-02-14 14:57:40 INFO launcher-proc-1:SparkShellApp:61 - at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scal
a:137)
2019-02-14 14:57:40 INFO launcher-proc-1:SparkShellApp:61 - at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala
)
2019-02-14 14:57:40 INFO launcher-proc-1:SparkShellApp:61 - 2019-02-14 14:57:40 INFO Thread-21:AnnotationConfigEmbeddedWebA
pplicationContext:984 - Closing org.springframework.boot.context.embedded.AnnotationConfigEmbeddedWebApplicationContext@5ddea
849: startup date [Thu Feb 14 14:57:16 CST 2019]; root of context hierarchy

Environment

None

Status

Assignee

Unassigned

Reporter

Lin Forest

Labels

None

Priority

High
Configure