I am trying to create vectorAssembler to create input for logistic regression and use the following code:
//imports import org.apache.spark.ml.feature.VectorAssembler import org.apache.spark.mllib.linalg.{Vector, Vectors, VectorUDT} 1 val assembler = new VectorAssembler() 2 .setInputCols(flattenedPath.columns.diff(Seq("userid", "Conversion"))) 3 .setOutputCol("features") 4 val output = assembler.transform(flattenedPath) 5 println(output.select("features", "Conversion").first())
I get the following exception on line 4:
Exception in thread "main" java.lang.RuntimeException: error reading Scala signature of org.apache.spark.mllib.linalg.Vector: assertion failed: unsafe symbol SparseVector (child of package linalg) in runtime reflection universe at scala.reflect.internal.pickling.UnPickler.unpickle(UnPickler.scala:46) at scala.reflect.runtime.JavaMirrors$JavaMirror.unpickleClass(JavaMirrors.scala:619) at scala.reflect.runtime.SymbolLoaders$TopClassCompleter$$anonfun$complete$1.apply$mcV$sp(SymbolLoaders.scala:28) at scala.reflect.runtime.SymbolLoaders$TopClassCompleter$$anonfun$complete$1.apply(SymbolLoaders.scala:25) at scala.reflect.runtime.SymbolLoaders$TopClassCompleter$$anonfun$complete$1.apply(SymbolLoaders.scala:25) at scala.reflect.internal.SymbolTable.slowButSafeEnteringPhaseNotLaterThan(SymbolTable.scala:263) at scala.reflect.runtime.SymbolLoaders$TopClassCompleter.complete(SymbolLoaders.scala:25) at scala.reflect.runtime.SymbolLoaders$TopClassCompleter.load(SymbolLoaders.scala:33) at scala.reflect.runtime.SynchronizedSymbols$SynchronizedSymbol$$anonfun$typeParams$1.apply(SynchronizedSymbols.scala:140) at scala.reflect.runtime.SynchronizedSymbols$SynchronizedSymbol$$anonfun$typeParams$1.apply(SynchronizedSymbols.scala:133) at scala.reflect.runtime.Gil$class.gilSynchronized(Gil.scala:19) at scala.reflect.runtime.JavaUniverse.gilSynchronized(JavaUniverse.scala:16) at scala.reflect.runtime.SynchronizedSymbols$SynchronizedSymbol$class.gilSynchronizedIfNotThreadsafe(SynchronizedSymbols.scala:123) at scala.reflect.runtime.SynchronizedSymbols$SynchronizedSymbol$$anon$8.gilSynchronizedIfNotThreadsafe(SynchronizedSymbols.scala:168) at scala.reflect.runtime.SynchronizedSymbols$SynchronizedSymbol$class.typeParams(SynchronizedSymbols.scala:132) at scala.reflect.runtime.SynchronizedSymbols$SynchronizedSymbol$$anon$8.typeParams(SynchronizedSymbols.scala:168) at scala.reflect.internal.Types$NoArgsTypeRef.typeParams(Types.scala:1926) at scala.reflect.internal.Types$NoArgsTypeRef.isHigherKinded(Types.scala:1925) at scala.reflect.internal.transform.UnCurry$class.scala$reflect$internal$transform$UnCurry$$expandAlias(UnCurry.scala:22) at scala.reflect.internal.transform.UnCurry$$anon$2.apply(UnCurry.scala:26) at scala.reflect.internal.transform.Transforms$class.transformedType(Transforms.scala:43) at scala.reflect.internal.SymbolTable.transformedType(SymbolTable.scala:16) at scala.reflect.internal.Types$TypeApiImpl.erasure(Types.scala:225) at scala.reflect.internal.Types$TypeApiImpl.erasure(Types.scala:218)
I am using spark-mllib_2.11 1.6.0 jar
Any pointers on how to solve this problem.
source share