Object

com.microsoft.ml.spark

LightGBMUtils

Related Doc: package spark

Permalink

object LightGBMUtils

Helper utilities for LightGBM learners

Linear Supertypes
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. LightGBMUtils
  2. AnyRef
  3. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Value Members

  1. final def !=(arg0: Any): Boolean

    Permalink
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int

    Permalink
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean

    Permalink
    Definition Classes
    AnyRef → Any
  4. final def asInstanceOf[T0]: T0

    Permalink
    Definition Classes
    Any
  5. def clone(): AnyRef

    Permalink
    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  6. def createDriverNodesThread(numWorkers: Int, df: DataFrame, log: Logger, timeout: Double): (String, Int, Future[Unit])

    Permalink

    Opens a socket communications channel on the driver, starts a thread that waits for the host:port from the executors, and then sends back the information to the executors.

    Opens a socket communications channel on the driver, starts a thread that waits for the host:port from the executors, and then sends back the information to the executors.

    numWorkers

    The total number of training workers to wait for.

    returns

    The address and port of the driver socket.

  7. final def eq(arg0: AnyRef): Boolean

    Permalink
    Definition Classes
    AnyRef
  8. def equals(arg0: Any): Boolean

    Permalink
    Definition Classes
    AnyRef → Any
  9. def featurizeData(dataset: Dataset[_], labelColumn: String, featuresColumn: String): PipelineModel

    Permalink
  10. def finalize(): Unit

    Permalink
    Attributes
    protected[java.lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )
  11. def generateData(numRows: Int, rowsAsDoubleArray: Array[Array[Double]]): (SWIGTYPE_p_void, SWIGTYPE_p_double)

    Permalink
  12. def generateDenseDataset(numRows: Int, rowsAsDoubleArray: Array[Array[Double]]): SWIGTYPE_p_void

    Permalink
  13. def generateSparseDataset(sparseRows: Array[SparseVector]): SWIGTYPE_p_void

    Permalink

    Generates a sparse dataset in CSR format.

    Generates a sparse dataset in CSR format.

    sparseRows

    The rows of sparse vector.

  14. def getBoosterPtrFromModelString(lgbModelString: String): SWIGTYPE_p_void

    Permalink
  15. def getCategoricalIndexes(df: DataFrame, featuresCol: String, categoricalColumnIndexes: Array[Int], categoricalColumnSlotNames: Array[String]): Array[Int]

    Permalink
  16. final def getClass(): Class[_]

    Permalink
    Definition Classes
    AnyRef → Any
  17. def getExecutors(dataset: Dataset[_], numCoresPerExec: Int): Array[(Int, String)]

    Permalink

    Returns a list of executor id and host.

    Returns a list of executor id and host.

    dataset

    The dataset containing the current spark session.

    numCoresPerExec

    The number of cores per executor.

    returns

    List of executors as an array of (id,host).

  18. def getHostToIP(hostname: String): String

    Permalink
  19. def getId(): Int

    Permalink

    Returns an integer ID for the current node.

    Returns an integer ID for the current node.

    returns

    In cluster, returns the executor id. In local case, returns the worker id.

  20. def getNodes(data: DataFrame, defaultListenPort: Int, numCoresPerExec: Int): Array[(Int, String)]

    Permalink

    Returns the executor node ips and ports.

    Returns the executor node ips and ports.

    data

    The input dataframe.

    defaultListenPort

    The default listen port.

    numCoresPerExec

    The number of cores per executor.

    returns

    List of nodes as comma separated string and count.

  21. def getNodesFromPartitions(processedData: DataFrame, defaultListenPort: Int, executorToHost: Map[Int, String]): Array[(Int, String)]

    Permalink

    Returns the nodes from mapPartitions.

    Returns the nodes from mapPartitions. Only run in case when num partitions < num executors.

    processedData

    The input data.

    defaultListenPort

    The default listening port.

    executorToHost

    Map from executor id to host name.

    returns

    The list of nodes in host:port format.

  22. def getNodesFromPartitionsLocal(processedData: DataFrame, defaultListenPort: Int): Array[(Int, String)]

    Permalink

    Returns the nodes from mapPartitions.

    Returns the nodes from mapPartitions. Only run in local[*] case.

    processedData

    The input data.

    defaultListenPort

    The default listening port.

    returns

    The list of nodes in host:port format.

  23. def getNumCoresPerExecutor(dataset: Dataset[_]): Int

    Permalink

    Get number of cores from dummy dataset for 1 executor.

    Get number of cores from dummy dataset for 1 executor. Note: all executors have same number of cores, and this is more reliable than getting value from conf.

    dataset

    The dataset containing the current spark session.

    returns

    The number of cores per executor.

  24. def getNumExecutorCores(dataset: Dataset[_], numCoresPerExec: Int): Int

    Permalink

    Returns the number of executors * number of cores.

    Returns the number of executors * number of cores.

    dataset

    The dataset containing the current spark session.

    numCoresPerExec

    The number of cores per executor.

    returns

    The number of executors * number of cores.

  25. def hashCode(): Int

    Permalink
    Definition Classes
    AnyRef → Any
  26. def initializeNativeLibrary(): Unit

    Permalink

    Loads the native shared object binaries lib_lightgbm.so and lib_lightgbm_swig.so

  27. def intToPtr(value: Int): SWIGTYPE_p_int64_t

    Permalink
  28. final def isInstanceOf[T0]: Boolean

    Permalink
    Definition Classes
    Any
  29. final def ne(arg0: AnyRef): Boolean

    Permalink
    Definition Classes
    AnyRef
  30. def newDoubleArray(array: Array[Double]): (SWIGTYPE_p_void, SWIGTYPE_p_double)

    Permalink
  31. def newIntArray(array: Array[Int]): (SWIGTYPE_p_int32_t, SWIGTYPE_p_int)

    Permalink
  32. final def notify(): Unit

    Permalink
    Definition Classes
    AnyRef
  33. final def notifyAll(): Unit

    Permalink
    Definition Classes
    AnyRef
  34. final def synchronized[T0](arg0: ⇒ T0): T0

    Permalink
    Definition Classes
    AnyRef
  35. def toAddr(hostAndId: (String, Int), defaultListenPort: Int): String

    Permalink

    Converts a host,id pair to the lightGBM host:port format.

    Converts a host,id pair to the lightGBM host:port format.

    hostAndId

    The host,id.

    defaultListenPort

    The default listen port.

    returns

    The string lightGBM representation of host:port.

  36. def toString(): String

    Permalink
    Definition Classes
    AnyRef → Any
  37. def validate(result: Int, component: String): Unit

    Permalink
  38. final def wait(): Unit

    Permalink
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  39. final def wait(arg0: Long, arg1: Int): Unit

    Permalink
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  40. final def wait(arg0: Long): Unit

    Permalink
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )

Inherited from AnyRef

Inherited from Any

Members