fix date parsing
This commit is contained in:
@@ -11,8 +11,8 @@ object KMeans {
|
||||
|
||||
|
||||
def train(dataset : DataFrame, iterations:Int) : Unit = {
|
||||
val K = 10 // Number of desired clusters
|
||||
val relevantData = dataset.select("Reputation", "Views", "UpVotes", "DownVotes")
|
||||
val K = 4 // Number of desired clusters
|
||||
val relevantData = dataset.select("Reputation", "LastAccessDate")
|
||||
val m = relevantData.columns.length //number of features
|
||||
val rows = relevantData.rdd
|
||||
val rowsAsArray = rows.map(row => convertRow(row, m)).persist()
|
||||
|
||||
@@ -62,7 +62,7 @@ object XMLParser {
|
||||
*/
|
||||
private def GenerateSchemaFromString(schemaString: String, schemaType: Array[DataType]) : StructType = {
|
||||
// Replace all DateTypes with Longs as date will now be stored as longs.
|
||||
val sT = schemaType.map(i => if (i==DateType) LongType else i)
|
||||
val sT = schemaType.map(i => if (i==DateType) IntegerType else i)
|
||||
val schemaPairs = schemaString.split(" ") zip sT
|
||||
|
||||
// Create schema for columns and set their datatypes for DataFrame based on attribute names.
|
||||
|
||||
Reference in New Issue
Block a user