Basic RDD Transformations
1. MAP
val x = [Link](Array("b", "a", "c"))
val y = [Link](z => (z,1))
println([Link]().mkString(", "))
println([Link]().mkString(", "))
Outpu
t:
b, a, c
(b,1), (a,1), (c,1)
2. FILTER
val x = [Link](Array(1,2,3))
val y = [Link](n => n%2 == 1)
println([Link]().mkString(", "))
println([Link]().mkString(", "))
output:
1, 2, 3
1, 3
3. FLATMAP
val x = [Link](Array(1,2,3))
val y = [Link](n => Array(n, n*100, 42))
println([Link]().mkString(", "))
println([Link]().mkString(", "))
output:
1, 2, 3
1, 100, 42, 2, 200, 42, 3, 300, 42
4. GROUPBY
val x = [Link](
Array("John", "Fred", "Anna", "James"))
val y = [Link](w => [Link](0))
println([Link]().mkString(", "))
output:
['John', 'Fred', 'Anna', 'James']
[('A',['Anna']),('J',['John','James']),('F',['Fred'])]
5. GROUPBYKEY
val x = [Link](
Array(('B',5),('B',4),('A',3),('A',2),('A',1)))
val y = [Link]()
println([Link]().mkString(", "))
println([Link]().mkString(", "))
output:
[('B', 5),('B', 4),('A', 3),('A', 2),('A', 1)]
[('A', [2, 3, 1]),('B',[5, 4])
6. SAMPLE
val x = [Link](Array(1, 2, 3, 4, 5))
val y = [Link](false, 0.4)
// omitting seed will yield different output
println([Link]().mkString(", "))
output:
[1, 2, 3, 4, 5]
[1, 3]
7. UNION
val x = [Link](Array(1,2,3), 2)
val y = [Link](Array(3,4), 1)
val z = [Link](y)
val zOut = [Link]().collect()
output:
[1, 2, 3]
[3, 4]
[[1], [2, 3], [3, 4]]
8. JOIN
val x = [Link](Array(("a", 1), ("b", 2)))
val y = [Link](Array(("a", 3), ("a", 4), ("b", 5)))
val z = [Link](y)
println([Link]().mkString(", "))
output:
[("a", 1), ("b", 2)]
[("a", 3), ("a", 4), ("b", 5)]
[('a', (1, 3)), ('a', (1, 4)), ('b', (2, 5))]
9. DISTINCT
val x = [Link](Array(1,2,3,3,4))
val y = [Link]()
println([Link]().mkString(", "))
output:
[1, 2, 3, 3, 4]
[1, 2, 3, 4]
Basic RDD ACTIONS
1. COLLECT
val x = [Link](Array(1,2,3), 2)
val y = [Link]()
val xOut = [Link]().collect()
println(y)
output:
[[1], [2, 3]]
[1, 2, 3]
2. REDUCE
val x = [Link](Array(1,2,3,4))
val y = [Link]((a,b) => a+b)
println([Link](", "))
println(y)
output:
[1, 2, 3, 4]
10
3. AGGREGATE
val x = [Link](Array(1,2,3,4))
val y = [Link]((Array[Int](), 0))(seqOp, combOp)
println(y)
output:
[1, 2, 3, 4]
(Array(3, 1, 2, 4),10)
4. MAX
val x = [Link](Array(2,4,1))
val y = [Link]
println([Link]().mkString(", "))
println(y)
output:
[2, 4, 1]
4
5. SUM
val x = [Link](Array(2,4,1))
val y = [Link]
println([Link]().mkString(", "))
println(y)
output:
[2, 4, 1]
7
6. MEAN
val x = [Link](Array(2,4,1))
val y = [Link]
println([Link]().mkString(", "))
println(y)
output:
[2, 4, 1]
2.3333333
7. STDEV
val x = [Link](Array(2,4,1))
val y = [Link]
println([Link]().mkString(", "))
println(y)
output:
[2, 4, 1]
1.2472191