Data transformation feed fails with error: 'DataSet does not exist'
Description
1) Create a data transformation feed
2) Use Hive as the data source
3) Run the feed
The feed fails with this error:
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] Exception in thread "main" java.lang.IllegalArgumentException: DataSet does not exist: 7ffdce9e-5bef-446e-a115-926912c469f1
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at com.thinkbiganalytics.spark.shell.AbstractCatalogDataSetProvider.findById(AbstractCatalogDataSetProvider.java:76)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at com.thinkbiganalytics.spark.shell.AbstractCatalogDataSetProvider.read(AbstractCatalogDataSetProvider.java:106)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at $line3.$read$$iw$$iw$Script.eval(<console>:8)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at com.thinkbiganalytics.spark.repl.Script.run(Script.scala:17)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at $line3.$read$$iw$$iw$.<init>(<console>:10)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at $line3.$read$$iw$$iw$.<clinit>(<console>)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at $line3.$eval$.<init>(<console>:7)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at $line3.$eval$.<clinit>(<console>)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at $line3.$eval.$print(<console>)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at java.lang.reflect.Method.invoke(Method.java:498)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:734)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:983)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at scala.tools.nsc.interpreter.IMain.loadAndRunReq$1(IMain.scala:573)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:604)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:568)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at com.thinkbiganalytics.spark.repl.SparkScriptEngine.execute(SparkScriptEngine.java:179)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at com.thinkbiganalytics.spark.repl.ScriptEngine.eval(ScriptEngine.java:157)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at com.thinkbiganalytics.spark.interpreter.App.main(App.java:102)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at java.lang.reflect.Method.invoke(Method.java:498)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:738)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121)
2018-11-06 22:40:36,057 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
2018-11-06 22:40:36,070 INFO [stream error] c.t.nifi.v2.spark.ExecuteSparkJob ExecuteSparkJob[id=d4f7b3f3-eb75-3695-2d92-a90dcad16a64] 18/11/06 22:40:35 INFO SparkContext: Invoking stop() from shutdown hookEnvironment
None
Status
Assignee
Scott Reisdorf
Reporter
Jagrut Sharma
Labels
None
Reviewer
None
Story point estimate
None
Sprint
None
Fix versions
Affects versions
0.10.0
Priority
MediumConfigure