Pattern matching of Spark1.4 Source Code Walking Notes

Pattern matching in RDD:

def hasNext: Boolean = (thisIter.hasNext, otherIter.hasNext) match {

case (true, true) => true

case (false, false) => false

case _ => throw new SparkException("Can only zip RDDs with " +

"same number of elements in each partition")

}

jobResult = jobResult match {

case Some(value) => Some(f(value, taskResult.get))

case None => taskResult

}

take(1) match {

case Array(t) => t

case _ => throw new UnsupportedOperationException("empty collection")

}

The following is easier to understand:

val len = rdd.dependencies.length

len match {

case 0 => Seq.empty

case 1 =>

val d = rdd.dependencies.head

debugString(d.rdd, prefix, d.isInstanceOf[ShuffleDependency[_, _, _]], true)

case _ => //all in the bowl

val frontDeps = rdd.dependencies.take(len - 1)

val frontDepStrings = frontDeps.flatMap(

d => debugString(d.rdd, prefix, d.isInstanceOf[ShuffleDependency[_, _, _]]))

val lastDep = rdd.dependencies.last

val lastDepStrings =

debugString(lastDep.rdd, prefix, lastDep.isInstanceOf[ShuffleDependency[_, _, _]], true)

(frontDepStrings ++ lastDepStrings)

}