fix date parsing

This commit is contained in:
Joe Darby
2016-12-19 15:14:49 +00:00
parent 76950fb467
commit 5fd3555865
2 changed files with 3 additions and 3 deletions
+2 -2
View File
@@ -11,8 +11,8 @@ object KMeans {
def train(dataset : DataFrame, iterations:Int) : Unit = {
val K = 10 // Number of desired clusters
val relevantData = dataset.select("Reputation", "Views", "UpVotes", "DownVotes")
val K = 4 // Number of desired clusters
val relevantData = dataset.select("Reputation", "LastAccessDate")
val m = relevantData.columns.length //number of features
val rows = relevantData.rdd
val rowsAsArray = rows.map(row => convertRow(row, m)).persist()
+1 -1
View File
@@ -62,7 +62,7 @@ object XMLParser {
*/
private def GenerateSchemaFromString(schemaString: String, schemaType: Array[DataType]) : StructType = {
// Replace all DateTypes with Longs as date will now be stored as longs.
val sT = schemaType.map(i => if (i==DateType) LongType else i)
val sT = schemaType.map(i => if (i==DateType) IntegerType else i)
val schemaPairs = schemaString.split(" ") zip sT
// Create schema for columns and set their datatypes for DataFrame based on attribute names.