于2017-06-16由小牛君创建
val c = sc.parallelize(List("Gnu", "Cat", "Rat", "Dog", "Gnu", "Rat"), 2) res0: Array[String] = Array(Gnu, Cat, Rat, Dog, Gnu, Rat) |
def parallelize[T: ClassTag](
|
val rdd: RDD[Int] = sc.makeRDD(Array(1,2,3,4)) |
def makeRDD[T: ClassTag]( |
val rdd=sc.textFile("/hdfs/wordcount/in/words.txt").flatMap(_.split("\\s+")).map((_,1)).reduceByKey(_+_); rdd.saveAsTextFile("/hdfs/wordcount/out") rdd.saveAsTextFile("/hdfs/wordcount/out2",classOf[org.apache.hadoop.io.compress.GzipCodec]) |
def textFile( |
def saveAsTextFile(path: String): Unit = withScope { |
def saveAsTextFile(path: String, codec: Class[_ <: CompressionCodec]): Unit = withScope { |
val x = sc.parallelize(1 to 10, 3) res25: Array[Int] = Array(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) |
def saveAsObjectFile(path: String): Unit = withScope { |
def objectFile[T: ClassTag]( |
val v = sc.parallelize(Array(("owl",3), ("gnu",4), ("dog",1), ("cat",2), ("ant",5)), 2) val y = sc.sequenceFile[String,Int]("/hdfs/obj/seq_file") res31: Array[(String, Int)] = Array((owl,3), (gnu,4), (dog,1), (cat,2), (ant,5)) |
源码: |
def saveAsSequenceFile( |
def sequenceFile[K, V] |
在线咨询
免费热线
资料发放
技术答疑
关注微信