bp/Big_Data_Assignment_2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'joe-dev2' of https://github.com/Pezz89/Big_Data_Assignment_2 into joe-dev2

Browse Source
This commit is contained in:
Joe Darby
2016-12-19 13:41:38 +00:00
parent 498c6ba67a 6b31903c02
commit 2ef677443b
2 changed files with 6 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.idea/workspace.xml
Generated
+6 -8
View File
@@ -2,9 +2,7 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="a45ae16c-c18b-46fd-bdd5-74c3ba5fabef" name="Default" comment="">
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/src/main/scala/KMeans.scala" afterPath="$PROJECT_DIR$/src/main/scala/KMeans.scala" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/src/main/scala/XMLParser.scala" afterPath="$PROJECT_DIR$/src/main/scala/XMLParser.scala" />
</list>
<ignored path="$PROJECT_DIR$/target/" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
@@ -23,8 +21,8 @@
<file leaf-file-name="KMeans.scala" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/main/scala/KMeans.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="491">
<caret line="52" column="0" lean-forward="true" selection-start-line="52" selection-start-column="0" selection-end-line="52" selection-end-column="0" />
<state relative-caret-position="162">
<caret line="9" column="6" lean-forward="true" selection-start-line="9" selection-start-column="6" selection-end-line="9" selection-end-column="6" />
<folding>
<element signature="e#23#54#0" expanded="true" />
</folding>
@@ -256,12 +254,12 @@
<workItem from="1482003919097" duration="1224000" />
<workItem from="1482071638456" duration="2869000" />
<workItem from="1482150691487" duration="1272000" />
<workItem from="1482152647060" duration="2108000" />
<workItem from="1482152647060" duration="2239000" />
</task>
<servers />
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="15757000" />
<option name="totallyTimeSpent" value="15888000" />
</component>
<component name="ToolWindowManager">
<frame x="75" y="52" width="1605" height="893" extended-state="0" />
@@ -498,8 +496,8 @@
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/KMeans.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="491">
<caret line="52" column="0" lean-forward="true" selection-start-line="52" selection-start-column="0" selection-end-line="52" selection-end-column="0" />
<state relative-caret-position="162">
<caret line="9" column="6" lean-forward="true" selection-start-line="9" selection-start-column="6" selection-end-line="9" selection-end-column="6" />
<folding>
<element signature="e#23#54#0" expanded="true" />
</folding>
src/main/scala/KMeans.scala
-19
View File
@@ -3,7 +3,6 @@ package ClusterSOData
import org.apache.spark.rdd.RDD
import org.apache.spark.sql._
import scala.collection.mutable.ArrayBuffer
object KMeans {
/**
@@ -52,24 +51,6 @@ object KMeans {
}
/*def clustering(centres :Array[Array[Float]], rowsAsArray : RDD[Array[Float]], m : Int, K : Int) : Array[Array[Float]] = {
val clusterMap :RDD[(Int,Array[Float])]= rowsAsArray.map(row => (assignCluster(row,centres,m,K),row)).persist()
val newCentres = clusterMap.reduceByKey((a,b) => getMeanVector(a,b,m))
val arrayNewCentres = newCentres.collect()
var results = new Array[Array[Float]](K)
for ((i,x) <- arrayNewCentres) {
results(i) = x
}
//Check all results are valid (no null)
for (i <- 0 until K) {
if (results(i) == null) {
return null
}
}
return results
}*/
def clustering(centres :Array[Array[Float]], rowsAsArray : RDD[Array[Float]], m : Int, K : Int) : RDD[(Int,Array[Float])] = {
val clusterMap :RDD[(Int,Array[Float])]= rowsAsArray.map(row => (assignCluster(row,centres,m,K),row))
return clusterMap