LightGBMUtils

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def createDriverNodesThread(numWorkers: Int, df: DataFrame, log: Logger, timeout: Double): (String, Int, Future[Unit])

Opens a socket communications channel on the driver, starts a thread that waits for the host:port from the executors, and then sends back the information to the executors.
Opens a socket communications channel on the driver, starts a thread that waits for the host:port from the executors, and then sends back the information to the executors.
numWorkers
The total number of training workers to wait for.
returns
The address and port of the driver socket.
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def featurizeData(dataset: Dataset[_], labelColumn: String, featuresColumn: String): PipelineModel
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def generateData(numRows: Int, rowsAsDoubleArray: Array[Array[Double]]): (SWIGTYPE_p_void, SWIGTYPE_p_double)
def generateDenseDataset(numRows: Int, rowsAsDoubleArray: Array[Array[Double]], referenceDataset: Option[SWIGTYPE_p_void]): SWIGTYPE_p_void
def generateSparseDataset(sparseRows: Array[SparseVector], referenceDataset: Option[SWIGTYPE_p_void]): SWIGTYPE_p_void

Generates a sparse dataset in CSR format.
Generates a sparse dataset in CSR format.
sparseRows
The rows of sparse vector.
def getBoosterPtrFromModelString(lgbModelString: String): SWIGTYPE_p_void
def getCategoricalIndexes(df: DataFrame, featuresCol: String, categoricalColumnIndexes: Array[Int], categoricalColumnSlotNames: Array[String]): Array[Int]
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getExecutors(dataset: Dataset[_], numCoresPerExec: Int): Array[(Int, String)]

Returns a list of executor id and host.
Returns a list of executor id and host.
dataset
The dataset containing the current spark session.
numCoresPerExec
The number of cores per executor.
returns
List of executors as an array of (id,host).
def getHostToIP(hostname: String): String
def getId(): Int

Returns an integer ID for the current node.
Returns an integer ID for the current node.
returns
In cluster, returns the executor id. In local case, returns the worker id.
def getNodes(data: DataFrame, defaultListenPort: Int, numCoresPerExec: Int): Array[(Int, String)]

Returns the executor node ips and ports.
Returns the executor node ips and ports.
data
The input dataframe.
defaultListenPort
The default listen port.
numCoresPerExec
The number of cores per executor.
returns
List of nodes as comma separated string and count.
def getNodesFromPartitions(processedData: DataFrame, defaultListenPort: Int, executorToHost: Map[Int, String]): Array[(Int, String)]

Returns the nodes from mapPartitions.
Returns the nodes from mapPartitions. Only run in case when num partitions < num executors.
processedData
The input data.
defaultListenPort
The default listening port.
executorToHost
Map from executor id to host name.
returns
The list of nodes in host:port format.
def getNodesFromPartitionsLocal(processedData: DataFrame, defaultListenPort: Int): Array[(Int, String)]

Returns the nodes from mapPartitions.
Returns the nodes from mapPartitions. Only run in local[*] case.
processedData
The input data.
defaultListenPort
The default listening port.
returns
The list of nodes in host:port format.
def getNumCoresPerExecutor(dataset: Dataset[_]): Int

Get number of cores from dummy dataset for 1 executor.
Get number of cores from dummy dataset for 1 executor. Note: all executors have same number of cores, and this is more reliable than getting value from conf.
dataset
The dataset containing the current spark session.
returns
The number of cores per executor.
def getNumExecutorCores(dataset: Dataset[_], numCoresPerExec: Int): Int

Returns the number of executors * number of cores.
Returns the number of executors * number of cores.
dataset
The dataset containing the current spark session.
numCoresPerExec
The number of cores per executor.
returns
The number of executors * number of cores.
def hashCode(): Int

Definition Classes
AnyRef → Any
def initializeNativeLibrary(): Unit

Loads the native shared object binaries lib_lightgbm.so and lib_lightgbm_swig.so
def intToPtr(value: Int): SWIGTYPE_p_int64_t
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def newDoubleArray(array: Array[Double]): (SWIGTYPE_p_void, SWIGTYPE_p_double)
def newIntArray(array: Array[Int]): (SWIGTYPE_p_int32_t, SWIGTYPE_p_int)
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toAddr(hostAndId: (String, Int), defaultListenPort: Int): String

Converts a host,id pair to the lightGBM host:port format.
Converts a host,id pair to the lightGBM host:port format.
hostAndId
The host,id.
defaultListenPort
The default listen port.
returns
The string lightGBM representation of host:port.
def toString(): String

Definition Classes
AnyRef → Any
def validate(result: Int, component: String): Unit
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package spark

object LightGBMUtils

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

def createDriverNodesThread(numWorkers: Int, df: DataFrame, log: Logger, timeout: Double): (String, Int, Future[Unit])

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def featurizeData(dataset: Dataset[_], labelColumn: String, featuresColumn: String): PipelineModel

def finalize(): Unit

def generateData(numRows: Int, rowsAsDoubleArray: Array[Array[Double]]): (SWIGTYPE_p_void, SWIGTYPE_p_double)

def generateDenseDataset(numRows: Int, rowsAsDoubleArray: Array[Array[Double]], referenceDataset: Option[SWIGTYPE_p_void]): SWIGTYPE_p_void

def generateSparseDataset(sparseRows: Array[SparseVector], referenceDataset: Option[SWIGTYPE_p_void]): SWIGTYPE_p_void

def getBoosterPtrFromModelString(lgbModelString: String): SWIGTYPE_p_void

def getCategoricalIndexes(df: DataFrame, featuresCol: String, categoricalColumnIndexes: Array[Int], categoricalColumnSlotNames: Array[String]): Array[Int]

final def getClass(): Class[_]

def getExecutors(dataset: Dataset[_], numCoresPerExec: Int): Array[(Int, String)]

def getHostToIP(hostname: String): String

def getId(): Int

def getNodes(data: DataFrame, defaultListenPort: Int, numCoresPerExec: Int): Array[(Int, String)]

def getNodesFromPartitions(processedData: DataFrame, defaultListenPort: Int, executorToHost: Map[Int, String]): Array[(Int, String)]

def getNodesFromPartitionsLocal(processedData: DataFrame, defaultListenPort: Int): Array[(Int, String)]

def getNumCoresPerExecutor(dataset: Dataset[_]): Int

def getNumExecutorCores(dataset: Dataset[_], numCoresPerExec: Int): Int

def hashCode(): Int

def initializeNativeLibrary(): Unit

def intToPtr(value: Int): SWIGTYPE_p_int64_t

final def isInstanceOf[T0]: Boolean

final def ne(arg0: AnyRef): Boolean

def newDoubleArray(array: Array[Double]): (SWIGTYPE_p_void, SWIGTYPE_p_double)

def newIntArray(array: Array[Int]): (SWIGTYPE_p_int32_t, SWIGTYPE_p_int)

final def notify(): Unit

final def notifyAll(): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def toAddr(hostAndId: (String, Int), defaultListenPort: Int): String

def toString(): String

def validate(result: Int, component: String): Unit

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from AnyRef

Inherited from Any

Members