bp/Big_Data_Assignment_2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

changed kmeans

Browse Source
This commit is contained in:
Joe Darby
2016-12-15 21:14:51 +00:00
parent bd0af2fd2b
commit 26155067a8
5 changed files with 384 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.idea/misc.xml
Generated
+17 -1
View File
@@ -45,7 +45,23 @@
</list>
</option>
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" default="true" project-jdk-name="1.8 (1)" project-jdk-type="JavaSDK">
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" default="true" project-jdk-name="1.8" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
<component name="masterDetails">
<states>
<state key="ProjectJDKs.UI">
<settings>
<last-edited>1.8</last-edited>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
</states>
</component>
</project>
.idea/workspace.xml
Generated
+344 -102
View File
@@ -1,9 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="b41a9788-25b3-4e04-923f-17cde259631b" name="Default" comment="">
<list default="true" id="74fa95ce-dfd4-40da-a7a1-b336badfaea8" name="Default" comment="">
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/.idea/misc.xml" afterPath="$PROJECT_DIR$/.idea/misc.xml" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/src/main/scala/KMeans.scala" afterPath="$PROJECT_DIR$/src/main/scala/KMeans.scala" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/src/main/scala/Main.scala" afterPath="$PROJECT_DIR$/src/main/scala/Main.scala" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/src/main/scala/XMLParser.scala" afterPath="$PROJECT_DIR$/src/main/scala/XMLParser.scala" />
</list>
<ignored path="$PROJECT_DIR$/target/" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
@@ -19,11 +22,11 @@
<component name="ExecutionTargetManager" SELECTED_TARGET="default_target" />
<component name="FileEditorManager">
<leaf SIDE_TABS_SIZE_LIMIT_KEY="300">
<file leaf-file-name="KMeans.scala" pinned="false" current-in-tab="true">
<file leaf-file-name="KMeans.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/KMeans.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="353">
<caret line="20" column="31" lean-forward="true" selection-start-line="20" selection-start-column="31" selection-end-line="20" selection-end-column="31" />
<state relative-caret-position="918">
<caret line="51" column="5" lean-forward="true" selection-start-line="51" selection-start-column="5" selection-end-line="51" selection-end-column="5" />
<folding>
<element signature="e#23#54#0" expanded="true" />
</folding>
@@ -31,23 +34,38 @@
</provider>
</entry>
</file>
<file leaf-file-name="Row.scala" pinned="false" current-in-tab="false">
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-catalyst_2.10/1.6.0/spark-catalyst_2.10-1.6.0-sources.jar!/org/apache/spark/sql/Row.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="324">
<caret line="38" column="34" lean-forward="false" selection-start-line="38" selection-start-column="34" selection-end-line="38" selection-end-column="34" />
<folding>
<element signature="n#!!doc" expanded="false" />
<element signature="e#832#872#0" expanded="false" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="XMLParser.scala" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/main/scala/XMLParser.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="232">
<caret line="13" column="8" lean-forward="true" selection-start-line="13" selection-start-column="8" selection-end-line="13" selection-end-column="8" />
<folding>
<element signature="e#23#59#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="Main.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/Main.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="209">
<caret line="25" column="0" lean-forward="true" selection-start-line="25" selection-start-column="0" selection-end-line="25" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="XMLParser.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/XMLParser.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="396">
<caret line="52" column="3" lean-forward="true" selection-start-line="52" selection-start-column="3" selection-end-line="52" selection-end-column="3" />
<state relative-caret-position="180">
<caret line="10" column="3" lean-forward="true" selection-start-line="10" selection-start-column="3" selection-end-line="10" selection-end-column="3" />
<folding>
<element signature="e#23#59#0" expanded="true" />
<element signature="e#22#58#0" expanded="true" />
</folding>
</state>
</provider>
@@ -66,23 +84,23 @@
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/src/main/scala/XMLParser.scala" />
<option value="$PROJECT_DIR$/src/main/scala/Main.scala" />
<option value="$PROJECT_DIR$/src/main/scala/KMeans.scala" />
</list>
</option>
</component>
<component name="MavenImportPreferences">
<option name="importingSettings">
<MavenImportingSettings>
<option name="importAutomatically" value="true" />
</MavenImportingSettings>
</option>
<component name="JsBuildToolGruntFileManager" detection-done="true" sorting="DEFINITION_ORDER" />
<component name="JsBuildToolPackageJson" detection-done="true" sorting="DEFINITION_ORDER" />
<component name="JsGulpfileManager">
<detection-done>true</detection-done>
<sorting>DEFINITION_ORDER</sorting>
</component>
<component name="ProjectFrameBounds">
<option name="x" value="75" />
<option name="y" value="80" />
<option name="width" value="1275" />
<option name="height" value="611" />
<option name="x" value="87" />
<option name="y" value="81" />
<option name="width" value="1400" />
<option name="height" value="893" />
</component>
<component name="ProjectView">
<navigator currentView="ProjectPane" proportions="" version="1">
@@ -99,7 +117,6 @@
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="Scratches" />
<pane id="ProjectPane">
<subPane>
<PATH>
@@ -112,32 +129,6 @@
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="KMeans" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="Big_Data_Assignment_2" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="src" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="main" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="scala" />
<option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" />
</PATH_ELEMENT>
<PATH_ELEMENT>
<option name="myItemId" value="XMLParser.scala" />
<option name="myItemType" value="org.jetbrains.plugins.scala.components.ScalaDefsProjectViewProvider$ScalaFileTreeNode" />
</PATH_ELEMENT>
</PATH>
<PATH>
<PATH_ELEMENT>
<option name="myItemId" value="KMeans" />
@@ -162,10 +153,17 @@
</PATH>
</subPane>
</pane>
<pane id="PackagesPane" />
<pane id="Scratches" />
<pane id="Scope" />
<pane id="PackagesPane" />
</panes>
</component>
<component name="PropertiesComponent">
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="aspect.path.notification.shown" value="true" />
<property name="js.eslint.eslintPackage" value="" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
</component>
<component name="RunManager">
<configuration default="true" type="#org.jetbrains.idea.devkit.run.PluginConfigurationType" factoryName="Plugin">
<module name="" />
@@ -252,6 +250,136 @@
<envs />
<method />
</configuration>
<configuration default="true" type="ArquillianJUnit" factoryName="" nameIsGenerated="true">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<module name="" />
<option name="arquillianRunConfiguration">
<value>
<option name="containerStateName" value="" />
</value>
</option>
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
<option name="PACKAGE_NAME" />
<option name="MAIN_CLASS_NAME" />
<option name="METHOD_NAME" />
<option name="TEST_OBJECT" value="class" />
<option name="VM_PARAMETERS" />
<option name="PARAMETERS" />
<option name="WORKING_DIRECTORY" />
<option name="ENV_VARIABLES" />
<option name="PASS_PARENT_ENVS" value="true" />
<option name="TEST_SEARCH_SCOPE">
<value defaultName="singleModule" />
</option>
<envs />
<patterns />
<method />
</configuration>
<configuration default="true" type="ArquillianTestNG" factoryName="">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<module name="" />
<option name="arquillianRunConfiguration">
<value>
<option name="containerStateName" value="" />
</value>
</option>
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
<option name="SUITE_NAME" />
<option name="PACKAGE_NAME" />
<option name="MAIN_CLASS_NAME" />
<option name="METHOD_NAME" />
<option name="GROUP_NAME" />
<option name="TEST_OBJECT" value="CLASS" />
<option name="VM_PARAMETERS" />
<option name="PARAMETERS" />
<option name="WORKING_DIRECTORY" />
<option name="OUTPUT_DIRECTORY" />
<option name="ANNOTATION_TYPE" />
<option name="ENV_VARIABLES" />
<option name="PASS_PARENT_ENVS" value="true" />
<option name="TEST_SEARCH_SCOPE">
<value defaultName="singleModule" />
</option>
<option name="USE_DEFAULT_REPORTERS" value="false" />
<option name="PROPERTIES_FILE" />
<envs />
<properties />
<listeners />
<method />
</configuration>
<configuration default="true" type="Cold Fusion runner description" factoryName="Cold Fusion" custom_browser="" web_path="">
<method />
</configuration>
<configuration default="true" type="CucumberJavaRunConfigurationType" factoryName="Cucumber java">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<option name="myFilePath" />
<option name="GLUE" />
<option name="myNameFilter" />
<option name="myGeneratedName" />
<option name="MAIN_CLASS_NAME" />
<option name="VM_PARAMETERS" />
<option name="PROGRAM_PARAMETERS" />
<option name="WORKING_DIRECTORY" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
<option name="ALTERNATIVE_JRE_PATH" />
<option name="ENABLE_SWING_INSPECTOR" value="false" />
<option name="ENV_VARIABLES" />
<option name="PASS_PARENT_ENVS" value="true" />
<module name="" />
<envs />
<method />
</configuration>
<configuration default="true" type="FlashRunConfigurationType" factoryName="Flash App">
<option name="BCName" value="" />
<option name="IOSSimulatorSdkPath" value="" />
<option name="adlOptions" value="" />
<option name="airProgramParameters" value="" />
<option name="appDescriptorForEmulator" value="Android" />
<option name="debugTransport" value="USB" />
<option name="debuggerSdkRaw" value="BC SDK" />
<option name="emulator" value="NexusOne" />
<option name="emulatorAdlOptions" value="" />
<option name="fastPackaging" value="true" />
<option name="fullScreenHeight" value="0" />
<option name="fullScreenWidth" value="0" />
<option name="launchUrl" value="false" />
<option name="launcherParameters">
<LauncherParameters>
<option name="browser" value="a7bb68e0-33c0-4d6f-a81a-aac1fdb870c8" />
<option name="launcherType" value="OSDefault" />
<option name="newPlayerInstance" value="false" />
<option name="playerPath" value="/usr/bin/flashplayerdebugger" />
</LauncherParameters>
</option>
<option name="mobileRunTarget" value="Emulator" />
<option name="moduleName" value="" />
<option name="overriddenMainClass" value="" />
<option name="overriddenOutputFileName" value="" />
<option name="overrideMainClass" value="false" />
<option name="runTrusted" value="true" />
<option name="screenDpi" value="0" />
<option name="screenHeight" value="0" />
<option name="screenWidth" value="0" />
<option name="url" value="http://" />
<option name="usbDebugPort" value="7936" />
<method />
</configuration>
<configuration default="true" type="FlexUnitRunConfigurationType" factoryName="FlexUnit" appDescriptorForEmulator="Android" class_name="" emulatorAdlOptions="" method_name="" package_name="" scope="Class">
<option name="BCName" value="" />
<option name="launcherParameters">
<LauncherParameters>
<option name="browser" value="a7bb68e0-33c0-4d6f-a81a-aac1fdb870c8" />
<option name="launcherType" value="OSDefault" />
<option name="newPlayerInstance" value="false" />
<option name="playerPath" value="/usr/bin/flashplayerdebugger" />
</LauncherParameters>
</option>
<option name="moduleName" value="" />
<option name="trusted" value="true" />
<method />
</configuration>
<configuration default="true" type="GradleRunConfiguration" factoryName="Gradle">
<ExternalSystemSettings>
<option name="executionName" />
@@ -268,6 +396,15 @@
</ExternalSystemSettings>
<method />
</configuration>
<configuration default="true" type="GrailsRunConfigurationType" factoryName="Grails">
<setting name="vmparams" value="" />
<setting name="cmdLine" value="run-app" />
<setting name="passParentEnv" value="true" />
<setting name="launchBrowser" value="true" />
<setting name="launchBrowserUrl" value="" />
<setting name="depsClasspath" value="false" />
<method />
</configuration>
<configuration default="true" type="JUnit" factoryName="JUnit">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<module name="" />
@@ -310,6 +447,15 @@
<envs />
<method />
</configuration>
<configuration default="true" type="JavaScriptTestRunnerProtractor" factoryName="Protractor">
<config-file value="" />
<node-interpreter value="project" />
<envs />
<method />
</configuration>
<configuration default="true" type="JavascriptDebugType" factoryName="JavaScript Debug">
<method />
</configuration>
<configuration default="true" type="JetRunConfigurationType" factoryName="Kotlin">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<option name="MAIN_CLASS_NAME" />
@@ -373,6 +519,12 @@
<envs />
<method />
</configuration>
<configuration default="true" type="SpringBootApplicationConfigurationType" factoryName="Spring Boot">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<module name="" />
<envs />
<method />
</configuration>
<configuration default="true" type="TestNG" factoryName="TestNG">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<module name="" />
@@ -401,6 +553,28 @@
<listeners />
<method />
</configuration>
<configuration default="true" type="js.build_tools.gulp" factoryName="Gulp.js">
<node-interpreter>project</node-interpreter>
<node-options />
<gulpfile />
<tasks />
<arguments />
<envs />
<method />
</configuration>
<configuration default="true" type="js.build_tools.npm" factoryName="npm">
<command value="run" />
<scripts />
<node-interpreter value="project" />
<envs />
<method />
</configuration>
<configuration default="true" type="osgi.bnd.run" factoryName="Run Launcher">
<method />
</configuration>
<configuration default="true" type="osgi.bnd.run" factoryName="Test Launcher (JUnit)">
<method />
</configuration>
<configuration default="true" type="uTestRunConfiguration" factoryName="utest">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<module name="" />
@@ -427,16 +601,21 @@
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="b41a9788-25b3-4e04-923f-17cde259631b" name="Default" comment="" />
<created>1481799944130</created>
<changelist id="74fa95ce-dfd4-40da-a7a1-b336badfaea8" name="Default" comment="" />
<created>1481830590764</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1481799944130</updated>
<updated>1481830590764</updated>
<workItem from="1481830593703" duration="700000" />
<workItem from="1481831304788" duration="5133000" />
</task>
<servers />
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="5833000" />
</component>
<component name="ToolWindowManager">
<frame x="75" y="80" width="1275" height="611" extended-state="0" />
<frame x="87" y="81" width="1400" height="893" extended-state="0" />
<editor active="true" />
<layout>
<window_info id="Palette" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
@@ -444,6 +623,7 @@
<window_info id="Nl-Palette" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Palette&#9;" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Image Layers" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Java Enterprise" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Capture Analysis" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" />
<window_info id="Maven Projects" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
@@ -453,7 +633,8 @@
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Capture Tool" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.24941176" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="Database" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="UI Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
@@ -468,6 +649,9 @@
<window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
</layout>
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="processedProjectFiles" value="true" />
</component>
<component name="VcsContentAnnotationSettings">
<option name="myLimit" value="2678400000" />
</component>
@@ -478,8 +662,8 @@
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/src/main/scala/KMeans.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<state relative-caret-position="360">
<caret line="20" column="31" lean-forward="true" selection-start-line="20" selection-start-column="31" selection-end-line="20" selection-end-column="31" />
<folding>
<element signature="e#23#54#0" expanded="true" />
</folding>
@@ -488,26 +672,8 @@
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/XMLParser.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="270">
<caret line="15" column="7" lean-forward="false" selection-start-line="15" selection-start-column="7" selection-end-line="15" selection-end-column="7" />
<folding>
<element signature="e#23#59#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.jar!/scala/collection/TraversableLike.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="168">
<caret line="15" column="6" lean-forward="false" selection-start-line="15" selection-start-column="6" selection-end-line="15" selection-end-column="6" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/XMLParser.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="396">
<caret line="52" column="3" lean-forward="true" selection-start-line="52" selection-start-column="3" selection-end-line="52" selection-end-column="3" />
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding>
<element signature="e#23#59#0" expanded="true" />
</folding>
@@ -516,37 +682,113 @@
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/Main.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="209">
<caret line="25" column="0" lean-forward="true" selection-start-line="25" selection-start-column="0" selection-end-line="25" selection-end-column="0" />
<state relative-caret-position="108">
<caret line="13" column="7" lean-forward="false" selection-start-line="13" selection-start-column="7" selection-end-line="13" selection-end-column="7" />
<folding>
<element signature="e#22#58#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-sql_2.10/1.6.0/spark-sql_2.10-1.6.0.jar!/org/apache/spark/sql/DataFrame.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="36">
<caret line="2" column="6" lean-forward="false" selection-start-line="2" selection-start-column="6" selection-end-line="2" selection-end-column="6" />
<folding />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-catalyst_2.10/1.6.0/spark-catalyst_2.10-1.6.0.jar!/org/apache/spark/sql/Row.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="18">
<caret line="1" column="6" lean-forward="false" selection-start-line="1" selection-start-column="6" selection-end-line="1" selection-end-column="6" />
<folding />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-core_2.10/1.6.0/spark-core_2.10-1.6.0.jar!/org/apache/spark/rdd/PairRDDFunctions.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="162">
<caret line="9" column="6" lean-forward="false" selection-start-line="9" selection-start-column="6" selection-end-line="9" selection-end-column="6" />
<folding />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-core_2.10/1.6.0/spark-core_2.10-1.6.0-sources.jar!/org/apache/spark/rdd/PairRDDFunctions.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-2196">
<caret line="57" column="19" lean-forward="false" selection-start-line="57" selection-start-column="19" selection-end-line="57" selection-end-column="19" />
<folding>
<element signature="n#!!doc" expanded="false" />
<element signature="e#832#858#0" expanded="false" />
</folding>
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-catalyst_2.10/1.6.0/spark-catalyst_2.10-1.6.0-sources.jar!/org/apache/spark/sql/Row.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="324">
<caret line="38" column="34" lean-forward="false" selection-start-line="38" selection-start-column="34" selection-end-line="38" selection-end-column="34" />
<folding>
<element signature="n#!!doc" expanded="false" />
<element signature="e#832#872#0" expanded="false" />
</folding>
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5.jar!/scala/package.class">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="262">
<caret line="26" column="7" lean-forward="false" selection-start-line="26" selection-start-column="7" selection-end-line="26" selection-end-column="7" />
<folding />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/scala-lang/scala-library/2.10.5/scala-library-2.10.5-sources.jar!/scala/package.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="468">
<caret line="50" column="9" lean-forward="false" selection-start-line="50" selection-start-column="9" selection-end-line="50" selection-end-column="9" />
<folding />
</state>
</provider>
</entry>
<entry file="jar://$MAVEN_REPOSITORY$/org/apache/spark/spark-sql_2.10/1.6.0/spark-sql_2.10-1.6.0-sources.jar!/org/apache/spark/sql/DataFrame.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1260">
<caret line="112" column="3" lean-forward="false" selection-start-line="112" selection-start-column="3" selection-end-line="112" selection-end-column="3" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/KMeans.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="353">
<caret line="20" column="31" lean-forward="true" selection-start-line="20" selection-start-column="31" selection-end-line="20" selection-end-column="31" />
<state relative-caret-position="918">
<caret line="51" column="5" lean-forward="true" selection-start-line="51" selection-start-column="5" selection-end-line="51" selection-end-column="5" />
<folding>
<element signature="e#23#54#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</component>
<component name="masterDetails">
<states>
<state key="ProjectJDKs.UI">
<settings>
<last-edited>1.8 (1)</last-edited>
<splitter-proportions>
<option name="proportions">
<list>
<option value="0.2" />
</list>
</option>
</splitter-proportions>
</settings>
</state>
</states>
<entry file="file://$PROJECT_DIR$/src/main/scala/Main.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="180">
<caret line="10" column="3" lean-forward="true" selection-start-line="10" selection-start-column="3" selection-end-line="10" selection-end-column="3" />
<folding>
<element signature="e#22#58#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/XMLParser.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="232">
<caret line="13" column="8" lean-forward="true" selection-start-line="13" selection-start-column="8" selection-end-line="13" selection-end-column="8" />
<folding>
<element signature="e#23#59#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</component>
</project>
src/main/scala/KMeans.scala
+21 -7
View File
@@ -10,7 +10,7 @@ object KMeans {
*/
//Create a map to store each data row with its closest cluster index as key
def train(dataset : DataFrame) : RDD[(Int,ArrayBuffer[Float])] = {
def train(dataset : DataFrame) : RDD[(Int,Row)] = {
val rows = dataset.rdd
val K = 5 //number of intended clusters
val n = rows.count() //number of datapoints
@@ -53,14 +53,17 @@ object KMeans {
closestCentre
}
def calculateNewCentres(clusterMap : RDD[(Int,Row)]): RDD[(Int,ArrayBuffer[Float])] = {
val data = clusterMap.map(x => (x._1, x._2.asInstanceOf[ArrayBuffer[Float]]))
val newCentres = data.reduceByKey((a, b) => averageRow(a, b))
def calculateNewCentres(clusterMap : RDD[(Int,Row)]): RDD[(Int,Row)] = {
//val data = clusterMap.map(x => (x._1, x._2.asInstanceOf[ArrayBuffer[Double]]))
val newCentres = clusterMap.reduceByKey((a, b) => averageRow(a, b))
//val singleCluster = clusterMap.filter(x => x._1 == 0)
//val singleClusterAsArray = singleCluster.reduce()
newCentres
}
/*for (a <- 0 until K) {
var cluster = clusterMap.filter{case (a,_) => a == 0}
var data = cluster.map((_,a) => a :Row)*/
@@ -73,12 +76,23 @@ object KMeans {
return features
}*/
def averageRow(a :ArrayBuffer[Float], b:ArrayBuffer[Float]) : ArrayBuffer[Float] = {
val newRow = new ArrayBuffer[Float]()
for (i <- 0 until a.length) {
/*def averageRow(a :ArrayBuffer[Float], b:ArrayBuffer[Float]) : ArrayBuffer[Float] = {
val newRow = Row.apply()
for (i <- a.indices) {
val avgI = (a(i) + b(i)) /2
newRow(i) = avgI
}
newRow
}*/
def averageRow(a:Row, b:Row) : Row = {
val means = new ArrayBuffer[Double]()
for (i <- 0 until a.size) {
val mean = (a.getInt(i) + b.getInt(i)) /2.0
means(i) = mean
}
val newRow = Row.fromSeq(means)
newRow
}
}
src/main/scala/Main.scala
+1 -1
View File
@@ -22,7 +22,7 @@ object Main {
def main(args: Array[String]) {
// Retrieve data from StackOverflow dataset XMLs. Format into DataFrames
// for easy access to data elements.
val df = DataParser.ParseData()
val df = XMLParser.ParseData()
// get the users XML file
val users = df("users")
src/main/scala/XMLParser.scala
+1 -1
View File
@@ -13,7 +13,7 @@ import org.apache.spark.sql.types._
* Format and parse XML data to datasets, ready for further processing using
* spark
*/
object DataParser {
object XMLParser {
/*
* Generate array of DataFrames from XML content