import org.apache.spark.sql._
import org.apache.spark._
import org.apache.spark.sql.SQLContext
import java.util._
import java.text._
import com.databricks.spark.avro._
import java.sql.{Connection, DriverManager, ResultSet}
import org.apache.spark.sql._
import org.apache.spark.sql.{DataFrame, SparkSession}
object test_job {
val sc: SparkContext = new SparkContext(new SparkConf().setMaster("local[*]").setAppName("test_job"))
val sqlc = SparkSession.builder.appName("test_job").getOrCreate()
var ret : DataFrame = null
ret = sqlc.read
.format("com.databricks.spark.redshift")
.option("url", "jdbc:postgresql://**{servername}**.redshift.amazonaws.com:**{portnumber}**/prod")
.option("user", "**username**")
.option("password", "**password**")
.option("query", "select count(1) from datamart.train_ticket")
.option("forward_spark_s3_credentials", "true")
.load
println("Read data from redshift successfully")
ret.show()
}
這一段在Qubole上運行成功，但是回傳Empty，但是datamart.train_ticket這個表是有值的
import org.apache.spark.sql._
import com.databricks._
val sqlc = SparkSession.builder.appName("test_job").getOrCreate()
sqlc.read
.format("com.databricks.spark.redshift")
.option("url", "jdbc:postgresql://{servername}.redshift.amazonaws.com:{portnumber}/prod")
.option("user", "username")
.option("password", "password")
.option("query", "select train_nbr from datamart.train_ticket")
.option("forward_spark_s3_credentials", "true")
.option("tempdir", "s3n://{path}}/tempfile")
.load().createOrReplaceTempView("temp")
val allrecords = sqlc.sql("select count(1) from temp")
allrecords.show()
這一段Qubole運行失敗，報錯：java.lang.ClassNotFoundException: com.databricks.spark.redshift.DefaultSource
請大牛們幫忙看看
轉載請註明出處，本文鏈接：https://www.uj5u.com/qita/53484.html
標籤：Spark