Работа с Java и Spark Sql для чтения формата AVRO

Question

Как разрешить исключение, которое я получаю при запуске в eclipse ( noclassdeffounderror и classnotfoundexception) . Пробовал с другой версией блоков данных maven, но все равно получал ту же проблему.

AVRO файл присутствует в моей локальной системе. Ниже приведен полный код, который я использую.

Исключение:

Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/spark/sql/execution/datasources/FileFormat
    at java.lang.ClassLoader.defineClass1(Native Method)
    at java.lang.ClassLoader.defineClass(ClassLoader.java:763)
    at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
    at java.net.URLClassLoader.defineClass(URLClassLoader.java:467)
    at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
    at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
    at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
    at java.security.AccessController.doPrivileged(Native Method)
    at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
    at org.apache.spark.sql.execution.datasources.ResolvedDataSource$$anonfun$4$$anonfun$apply$1.apply(ResolvedDataSource.scala:62)
    at org.apache.spark.sql.execution.datasources.ResolvedDataSource$$anonfun$4$$anonfun$apply$1.apply(ResolvedDataSource.scala:62)
    at scala.util.Try$.apply(Try.scala:161)
    at org.apache.spark.sql.execution.datasources.ResolvedDataSource$$anonfun$4.apply(ResolvedDataSource.scala:62)
    at org.apache.spark.sql.execution.datasources.ResolvedDataSource$$anonfun$4.apply(ResolvedDataSource.scala:62)
    at scala.util.Try.orElse(Try.scala:82)
    at org.apache.spark.sql.execution.datasources.ResolvedDataSource$.lookupDataSource(ResolvedDataSource.scala:62)
    at org.apache.spark.sql.execution.datasources.ResolvedDataSource$.apply(ResolvedDataSource.scala:102)
    at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:119)
    at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:109)
    at example.orctest.main(orctest.java:25)
Caused by: java.lang.ClassNotFoundException: org.apache.spark.sql.execution.datasources.FileFormat
    at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
    at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
    at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
    ... 23 more

Ниже используется код:

package example;
import org.apache.hadoop.hive.ql.exec.spark.session.SparkSession;
import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.DataFrame;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.hive.HiveContext;
import org.apache.spark.sql.functions;
import org.apache.spark.sql.*;
import com.databricks.spark.avro.*;



public class orctest {

    public  static void main(String args[])
    {
        System.setProperty("hadoop.home.dir", "C:\\winutils\\");

        SparkConf sconf = new SparkConf().setAppName("Test").setMaster("local");
        //SparkSession spark = new SparkSession.builder().master("local").getOrCreate();
        JavaSparkContext jsc = new JavaSparkContext(sconf);
        SQLContext sqlContext = new SQLContext(jsc);
        DataFrame df1 = sqlContext.read().format("com.databricks.spark.avro").load("C:\\Users\\Desktop\\Spark\\part-m-00000.avro");

        sqlContext.registerDataFrameAsTable(df1, "customer");
        sqlContext.sql("Select * from customer").show();
        //df1.printSchema();

        jsc.stop();
    }

}

`

POM используется в проекте:

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>sparkcode</groupId>
  <artifactId>program1</artifactId>
  <version>0.0.1-SNAPSHOT</version>


<dependencies>


<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.10 -->
<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-core_2.10</artifactId>
    <version>1.6.0</version>
</dependency>
<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-sql_2.10</artifactId>
    <version>1.6.0</version>
</dependency>

<dependency>
    <groupId>org.apache.spark</groupId>
    <artifactId>spark-hive_2.10</artifactId>
    <version>1.6.0</version>
</dependency>
<dependency>

    <groupId>com.databricks</groupId>
    <artifactId>spark-avro_2.10</artifactId>
    <version>3.2.0</version>
</dependency>



<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client -->
<!-- <dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-client</artifactId>
    <version>2.8.1</version>
</dependency> -->



</dependencies>

</project>