spark

Type Members

1. class AddDocuments extends CognitiveServicesBase with HasCognitiveServiceInput with HasInternalJsonOutputParser with HasActionCol with HasServiceName with HasIndexName with HasBatchSize

   

2. class AdvancedRankingMetrics extends Serializable

   

3. class AnalyzeImage extends CognitiveServicesBase with HasImageInput with HasInternalJsonOutputParser with HasCognitiveServiceInput

   

4. class AssembleFeatures extends Estimator[AssembleFeaturesModel] with HasFeaturesCol with Wrappable with DefaultParamsWritable

   

   Creates a vector column of features from a collection of feature columns

5. class AssembleFeaturesModel extends Model[AssembleFeaturesModel] with Params with ConstructorWritable[AssembleFeaturesModel]

   

   Model produced by AssembleFeatures.

6. abstract class AsyncClient extends BaseClient

   

7. class AsyncHTTPClient extends AsyncClient with HTTPClient

   

8. abstract class AutoTrainedModel[TrainedModel <: Model[TrainedModel]] extends Model[TrainedModel] with ConstructorWritable[TrainedModel]

   

   Defines common inheritance and functions across auto trained models.

9. trait AutoTrainer[TrainedModel <: Model[TrainedModel]] extends Estimator[TrainedModel] with HasLabelCol with ComplexParamsWritable with HasFeaturesCol

   

   Defines common inheritance and parameters across trainers.

10. class BasicDatasetGenerationConstraints extends HasDatasetGenerationConstraints

    

    Basic constraints for generating a dataset.

11. class BatchIterator[T] extends Iterator[List[T]]

    

12. case class Benchmark(name: String, value: Double, precision: Double, higherIsBetter: Boolean = true) extends Product with Serializable

    

13. abstract class Benchmarks extends TestBase

    

14. class BestModel extends Model[BestModel] with ConstructorWritable[BestModel]

    

    Model produced by FindBestModel.

    Model produced by FindBestModel.

    Annotations

    @InternalWrapper()

15. case class BingImage(accentColor: String, contentSize: String, contentUrl: String, datePublished: String, encodingFormat: String, height: Int, hostPageDisplayUrl: String, hostPageUrl: String, id: String, imageId: String, imageInsightsToken: String, insightsMetadata: String, name: String, thumbnail: BingMediaSize, thumbnailUrl: String, webSearchUrl: String, width: Int) extends Product with Serializable

    

16. class BingImageSearch extends CognitiveServicesBase with HasCognitiveServiceInput with HasInternalJsonOutputParser

    

    Annotations

    @InternalWrapper()

17. case class BingImagesResponse(_type: String, instrumentation: BingInstrumentation, webSearchUrl: String, totalEstimatedMatches: Int, nextOffset: Int, value: Seq[BingImage], pivotSuggestions: Seq[BingPivot], queryExpansions: Seq[BingQuery], relatedSearches: Seq[BingQuery]) extends Product with Serializable

    

18. case class BingInstrumentation(_type: String) extends Product with Serializable

    

19. case class BingMediaSize(height: Int, width: Int) extends Product with Serializable

    

20. case class BingPivot(pivot: String, suggestions: Seq[BingQuery]) extends Product with Serializable

    

21. case class BingQuery(displayText: String, searchLink: String, text: String, thumbnail: BingThumbnail, webSearchUrl: String) extends Product with Serializable

    

22. case class BingThumbnail(thumbnailUrl: String) extends Product with Serializable

    

23. class Blur extends ImageTransformerStage

    

    Blurs the image using a box filter.

    Blurs the image using a box filter. The params are a map of the dimensions of the blurring box. Please refer to OpenCV for more information.

24. class BrainScriptBuilder extends AnyRef

    

    Utility methods for manipulating the BrainScript and overrides configs output to disk.

25. case class BrainScriptConfig(name: String, text: Seq[String]) extends Product with Serializable

    

26. class CNTKCommandBuilder extends CNTKCommandBuilderBase

    

27. abstract class CNTKCommandBuilderBase extends AnyRef

    

28. class CNTKConfig extends MMLConfig

    

29. class CNTKFunctionParam extends ComplexParam[SerializableFunction]

    

    Param for ByteArray.

    Param for ByteArray. Needed as spark has explicit params for many different types but not ByteArray.

30. class CNTKLearner extends Estimator[CNTKModel] with CNTKParams with Wrappable

    

    Annotations

    @InternalWrapper()

31. class CNTKModel extends Model[CNTKModel] with ComplexParamsWritable with HasMiniBatcher with Wrappable

    

    Annotations

    @InternalWrapper()

32. trait CNTKParams extends Params

    

33. class Cacher extends Transformer with Wrappable with DefaultParamsWritable

    

34. class CheckpointData extends Transformer with CheckpointDataParams

    

    Cache the dataset at this point to memory or memory and disk

35. trait CheckpointDataParams extends Wrappable with DefaultParamsWritable

    

36. class ClassBalancer extends Estimator[ClassBalancerModel] with DefaultParamsWritable with HasInputCol with HasOutputCol

    

    An estimator that calculates the weights for balancing a dataset.

    An estimator that calculates the weights for balancing a dataset. For example, if the negative class is half the size of the positive class, the weights will be 2 for rows with negative classes and 1 for rows with positive classes. these weights can be used in weighted classifiers and regressors to correct for heavilty skewed datasets. The inputCol should be the labels of the classes, and the output col will be the requisite weights.

37. class ClassBalancerModel extends Model[ClassBalancerModel] with ConstructorWritable[ClassBalancerModel]

    

38. case class ClassifierTrainParams(parallelism: String, numIterations: Int, learningRate: Double, numLeaves: Int, maxBin: Int, baggingFraction: Double, baggingFreq: Int, baggingSeed: Int, earlyStoppingRound: Int, featureFraction: Double, maxDepth: Int, minSumHessianInLeaf: Double, numMachines: Int, objective: String, modelString: Option[String], isUnbalance: Boolean, verbosity: Int, categoricalFeatures: Array[Int], numClass: Int, metric: String, boostFromAverage: Boolean, boostingType: String, lambdaL1: Double, lambdaL2: Double) extends TrainParams with Product with Serializable

    

    Defines the Booster parameters passed to the LightGBM classifier.

39. class CleanMissingData extends Estimator[CleanMissingDataModel] with HasInputCols with HasOutputCols with Wrappable with DefaultParamsWritable

    

    Removes missing values from input dataset.

    Removes missing values from input dataset. The following modes are supported: Mean - replaces missings with mean of fit column Median - replaces missings with approximate median of fit column Custom - replaces missings with custom value specified by user For mean and median modes, only numeric column types are supported, specifically: Int, Long, Float, Double For custom mode, the types above are supported and additionally: String, Boolean

40. class CleanMissingDataModel extends Model[CleanMissingDataModel] with ConstructorWritable[CleanMissingDataModel]

    

    Model produced by CleanMissingData.

41. class Cluster extends AnyRef

    

42. abstract class CognitiveServicesBase extends CognitiveServicesBaseWithoutHandler with HasHandler

    

43. abstract class CognitiveServicesBaseWithoutHandler extends Transformer with HTTPParams with HasOutputCol with HasURL with ComplexParamsWritable with HasSubscriptionKey with HasErrorCol

    

44. class ColorFormat extends ImageTransformerStage

    

    Converts an image from one color space to another, eg COLOR_BGR2GRAY.

    Converts an image from one color space to another, eg COLOR_BGR2GRAY. Refer to OpenCV for more information.

45. class ColumnNamesToFeaturize extends Serializable

    

    Class containing the list of column names to perform special featurization steps for.

    Class containing the list of column names to perform special featurization steps for. colNamesToHash - List of column names to hash. colNamesToDuplicateForMissings - List of column names containing doubles to duplicate so we can remove missing values from them. colNamesToTypes - Map of column names to their types. colNamesToCleanMissings - List of column names to clean missing values from (ignore). colNamesToVectorize - List of column names to vectorize using FastVectorAssembler. categoricalColumns - List of categorical columns to pass through or turn into indicator array. conversionColumnNamesMap - Map from old column names to new. addedColumnNamesMap - Map from old columns to newly generated columns for featurization.

    Annotations

    @SerialVersionUID()

46. class ComputeModelStatistics extends Transformer with ComputeModelStatisticsParams

    

    Evaluates the given scored dataset.

47. trait ComputeModelStatisticsParams extends Wrappable with DefaultParamsWritable with HasLabelCol with HasScoresCol with HasScoredLabelsCol with HasEvaluationMetric

    

48. class ComputePerInstanceStatistics extends Transformer with ComputePerInstanceStatisticsParams

    

    Evaluates the given scored dataset with per instance metrics.

    Evaluates the given scored dataset with per instance metrics.

    The Regression metrics are: - L1_loss - L2_loss

    The Classification metrics are: - log_loss

49. trait ComputePerInstanceStatisticsParams extends Wrappable with DefaultParamsWritable with HasLabelCol with HasScoresCol with HasScoredLabelsCol with HasScoredProbabilitiesCol with HasEvaluationMetric

    

50. abstract class Configuration extends AnyRef

    

51. class Consolidator[T] extends AnyRef

    

52. trait ConstructorReadable[T <: ConstructorWritable[_]] extends MLReadable[T]

    

53. class ConstructorReader[T] extends MLReader[T]

    

54. trait ConstructorWritable[S <: PipelineStage] extends MLWritable

    

    This trait allows you to easily add serialization to your Spark Models, assuming that they are completely parameterized by their constructor.

    This trait allows you to easily add serialization to your Spark Models, assuming that they are completely parameterized by their constructor. The main two fields required ate the TypeTag that allows the writer to inspect the constructor to get the types that need to be serialized, the actual objects that are serialized need to be defined in the field objectsToSave.

55. abstract class ConstructorWriter[S <: PipelineStage] extends MLWriter

    

56. class ContextObjectInputStream extends ObjectInputStream

    

57. class CropImage extends ImageTransformerStage

    

    Crops the image for processing.

    Crops the image for processing. The parameters are: "x" - First dimension; start of crop "y" - second dimension - start of crop "height" -height of cropped image "width" - width of cropped image "stageName" - "crop"

58. class CustomInputParser extends HTTPInputParser with ComplexParamsWritable

    

59. class CustomOutputParser extends HTTPOutputParser with ComplexParamsWritable

    

60. class DataConversion extends Transformer with Wrappable with DefaultParamsWritable

    

    Converts the specified list of columns to the specified type.

    Converts the specified list of columns to the specified type. Returns a new DataFrame with the converted columns

61. trait DataFrameEquality extends TestBase

    

62. case class DataFrameReaderExtensions2(dsr: DataFrameReader) extends Product with Serializable

    

63. case class DataFrameServingExtensions(df: DataFrame) extends Product with Serializable

    

64. class DataFrameSugars extends AnyRef

    

65. case class DataFrameWriterExtensions2[T](dsw: DataFrameWriter[T]) extends Product with Serializable

    

66. case class DataStreamReaderExtensions(dsr: DataStreamReader) extends Product with Serializable

    

67. case class DataStreamReaderExtensions2(dsr: DataStreamReader) extends Product with Serializable

    

68. case class DataStreamWriterExtensions[T](dsw: DataStreamWriter[T]) extends Product with Serializable

    

69. case class DataStreamWriterExtensions2[T](dsw: DataStreamWriter[T]) extends Product with Serializable

    

70. abstract class DataWriter extends AnyRef

    

71. case class DatasetMissingValuesGenerationOptions(percentMissing: Double, columnTypesWithMissings: ColumnOptions.ValueSet, dataTypesWithMissings: DataOptions.ValueSet) extends Product with Serializable

    

72. case class DatasetOptions(columnTypes: ColumnOptions.ValueSet, dataTypes: DataOptions.ValueSet, missingValuesOptions: DatasetMissingValuesGenerationOptions) extends Product with Serializable

    

    Options used to specify how a dataset will be generated.

    Options used to specify how a dataset will be generated. This contains information on what the data and column types (specified as flags) for generating a dataset will be limited to. It also contain options for all possible missing values generation and options for how values will be generated.

73. class DescribeImage extends CognitiveServicesBase with HasCognitiveServiceInput with HasImageInput with HasInternalJsonOutputParser

    

74. case class DetectEntitiesScore(id: String, entities: Seq[Entity]) extends Product with Serializable

    

75. class DetectFace extends CognitiveServicesBase with HasImageUrl with HasServiceParams with HasCognitiveServiceInput with HasInternalJsonOutputParser

    

76. case class DetectLanguageScore(id: String, detectedLanguages: Seq[DetectedLanguage]) extends Product with Serializable

    

77. case class DetectedLanguage(name: String, iso6391Name: String, score: Double) extends Product with Serializable

    

78. class DiscreteHyperParam[T] extends Dist[T]

    

79. abstract class Dist[T] extends AnyRef

    

    Represents a distribution of values.

    Represents a distribution of values.

    T

    The type T of the values generated.

80. class DoubleRangeHyperParam extends RangeHyperParam[Double]

    

81. class DropColumns extends Transformer with Wrappable with DefaultParamsWritable

    

    DropColumns takes a dataframe and a list of columns to drop as input and returns a dataframe comprised of only those columns not listed in the input list.

    DropColumns takes a dataframe and a list of columns to drop as input and returns a dataframe comprised of only those columns not listed in the input list.

82. class DynamicBufferedBatcher[T] extends Iterator[List[T]]

    

83. class DynamicMiniBatchTransformer extends Transformer with MiniBatchBase

    

84. class EnsembleByKey extends Transformer with Wrappable with DefaultParamsWritable

    

85. case class Entity(name: String, matches: Seq[Match], wikipediaLanguage: String, wikipediaId: String, wikipediaUrl: String, bingId: String) extends Product with Serializable

    

86. case class EntityData(content: Array[Byte], contentEncoding: Option[HeaderData], contentLength: Option[Long], contentType: Option[HeaderData], isChunked: Boolean, isRepeatable: Boolean, isStreaming: Boolean) extends Product with Serializable

    

87. class EntityDetector extends TextAnalyticsBase

    

88. class Explode extends Transformer with HasInputCol with HasOutputCol with Wrappable with DefaultParamsWritable

    

89. class Featurize extends Estimator[PipelineModel] with Wrappable with DefaultParamsWritable

    

    Featurizes a dataset.

    Featurizes a dataset. Converts the specified columns to feature columns.

90. case class Field(name: String, type: String, searchable: Boolean, filterable: Boolean, sortable: Boolean, facetable: Boolean, key: Boolean, retrievable: Boolean, analyzer: Option[String], searchAnalyzer: Option[String], indexAnalyzer: Option[String], synonymMaps: Option[String]) extends Product with Serializable

    

91. class FindBestModel extends Estimator[BestModel] with FindBestModelParams

    

    Evaluates and chooses the best model from a list of models.

    Evaluates and chooses the best model from a list of models.

    Annotations

    @InternalWrapper()

92. trait FindBestModelParams extends Wrappable with ComplexParamsWritable with HasEvaluationMetric

    

93. class FindSimilarFace extends CognitiveServicesBase with HasServiceParams with HasMaxNumOfCandidatesReturned with HasFaceIds with HasCognitiveServiceInput with HasInternalJsonOutputParser

    

94. class FixedBatcher[T] extends Iterator[List[T]]

    

95. class FixedBufferedBatcher[T] extends Iterator[List[T]]

    

96. class FixedMiniBatchTransformer extends Transformer with MiniBatchBase with HasBatchSize

    

97. class FlattenBatch extends Transformer with Wrappable with DefaultParamsWritable

    

98. class Flip extends ImageTransformerStage

    

    Flips the image

99. class FloatRangeHyperParam extends RangeHyperParam[Float]

    

100. class GaussianKernel extends ImageTransformerStage

     

     Applies gaussian kernel to blur the image.

     Applies gaussian kernel to blur the image. Please refer to OpenCV for detailed information about the parameters and their allowable values.

101. class GenerateDataType extends Serializable

     

     Generates the specified random data type.

102. class GenerateThumbnails extends CognitiveServicesBase with HasImageInput with HasWidth with HasHeight with HasSmartCropping with HasInternalJsonOutputParser with HasCognitiveServiceInput

     

103. class GridSpace extends ParamSpace

     

     Represents a parameter grid for tuning with discrete values.

     Represents a parameter grid for tuning with discrete values. Can be generated with the ParamGridBuilder.

104. class GroupFaces extends CognitiveServicesBase with HasServiceParams with HasFaceIds with HasCognitiveServiceInput with HasInternalJsonOutputParser

     

105. abstract class HTTPInputParser extends Transformer with HasOutputCol with HasInputCol

     

106. abstract class HTTPOutputParser extends Transformer with HasInputCol with HasOutputCol

     

107. trait HTTPParams extends Wrappable

     

108. case class HTTPRequestData(requestLine: RequestLineData, headers: Array[HeaderData], entity: Option[EntityData]) extends Product with Serializable

     

109. case class HTTPResponseData(headers: Array[HeaderData], entity: Option[EntityData], statusLine: StatusLineData, locale: String) extends Product with Serializable

     

110. class HTTPTransformer extends Transformer with HTTPParams with HasInputCol with HasOutputCol with HasHandler with ComplexParamsWritable

     

111. trait Handler extends AnyRef

     

112. trait HasActionCol extends HasServiceParams

     

113. trait HasBatchSize extends Params

     

114. trait HasCellSize extends Params

     

115. trait HasCognitiveServiceInput extends HasURL with HasSubscriptionKey

     

116. trait HasDatasetGenerationConstraints extends AnyRef

     

     Specifies the trait for constraints on generating a dataset.

117. trait HasDetectOrientation extends HasServiceParams

     

118. trait HasErrorCol extends Params

     

119. trait HasEvaluationMetric extends Wrappable

     

120. trait HasFaceIds extends HasServiceParams

     

121. trait HasFeaturesCol extends Wrappable

     

122. trait HasHandler extends Params

     

123. trait HasHeight extends HasServiceParams

     

124. trait HasImageBytes extends HasServiceParams

     

125. trait HasImageInput extends HasImageUrl with HasImageBytes with HasCognitiveServiceInput

     

126. trait HasImageUrl extends HasServiceParams

     

127. trait HasIndexName extends HasServiceParams

     

128. trait HasInputCol extends Wrappable

     

129. trait HasInputCols extends Wrappable

     

130. trait HasInternalJsonOutputParser extends AnyRef

     

131. trait HasLabelCol extends Wrappable

     

132. trait HasLanguage extends HasServiceParams

     

133. trait HasMaxNumOfCandidatesReturned extends HasServiceParams

     

134. trait HasMiniBatcher extends Params

     

135. trait HasModifier extends Params

     

136. trait HasOutputCol extends Wrappable

     

137. trait HasOutputCols extends Wrappable

     

138. trait HasScoredLabelsCol extends Wrappable

     

139. trait HasScoredProbabilitiesCol extends Wrappable

     

140. trait HasScoresCol extends Wrappable

     

141. trait HasServiceName extends HasServiceParams

     

142. trait HasServiceParams extends Params

     

143. trait HasSmartCropping extends HasServiceParams

     

144. trait HasSubscriptionKey extends HasServiceParams

     

145. trait HasURL extends Wrappable

     

146. trait HasValidationIndicatorCol extends Wrappable

     

147. trait HasWeightCol extends Wrappable

     

148. trait HasWidth extends HasServiceParams

     

149. class HdfsWriter extends SingleFileResolver

     

150. case class HeaderData(name: String, value: String) extends Product with Serializable

     

151. class HyperparamBuilder extends AnyRef

     

     Specifies the search space for hyperparameters.

152. class IdentifyFaces extends CognitiveServicesBase with HasServiceParams with HasMaxNumOfCandidatesReturned with HasFaceIds with HasCognitiveServiceInput with HasInternalJsonOutputParser

     

153. class ImageFeaturizer extends Transformer with HasInputCol with HasOutputCol with Wrappable with ComplexParamsWritable

     

     The ImageFeaturizer relies on a CNTK model to do the featurization, one can set this model using the modelLocation parameter.

     The ImageFeaturizer relies on a CNTK model to do the featurization, one can set this model using the modelLocation parameter. To map the nodes of the CNTK model onto the standard "layers" structure of a feed forward neural net, one needs to supply a list of node names that range from the output node, back towards the input node of the CNTK Function. This list does not need to be exhaustive, and is provided to you if you use a model downloaded from the ModelDownloader, one can find this layer list in the schema of the downloaded model.

     The ImageFeaturizer takes an input column of images (the type returned by the ImageReader), and automatically resizes them to fit the CMTKModel's inputs. It then feeds them through a pre-trained CNTK model. One can truncate the model using the cutOutputLayers parameter that determines how many layers to truncate from the output of the network. For example, layer=0 means that no layers are removed, layer=2 means that the image featurizer returns the activations of the layer that is two layers from the output layer.

     Annotations

     @InternalWrapper()

154. class ImageLIME extends Transformer with LIMEBase with Wrappable with HasModifier with HasCellSize

     

     Distributed implementation of Local Interpretable Model-Agnostic Explanations (LIME)

     Distributed implementation of Local Interpretable Model-Agnostic Explanations (LIME)

     https://arxiv.org/pdf/1602.04938v1.pdf

155. class ImageSetAugmenter extends Transformer with HasInputCol with HasOutputCol with DefaultParamsWritable

     

156. class ImageTransformer extends Transformer with HasInputCol with HasOutputCol with Wrappable with DefaultParamsWritable

     

     Image processing stage.

     Image processing stage. Please refer to OpenCV for additional information

     Annotations

     @InternalWrapper()

157. abstract class ImageTransformerStage extends Serializable

     

     Image processing stage.

158. trait IndexLister extends AnyRef

     

159. trait IndexParser extends AnyRef

     

160. case class IndexSchema(name: String, fields: Seq[Field]) extends Product with Serializable

     

161. class IndexToValue extends Transformer with HasInputCol with HasOutputCol with Wrappable with DefaultParamsWritable

     

     This class takes in a categorical column with MML style attibutes and then transforms it back to the original values.

     This class takes in a categorical column with MML style attibutes and then transforms it back to the original values. This extends MLLIB IndexToString by allowing the transformation back to any types of values.

162. case class InputData(format: String, path: String, shapes: Map[String, InputShape]) extends Product with Serializable

     

163. case class InputShape(dim: Int, form: String) extends Product with Serializable

     

164. class IntRangeHyperParam extends RangeHyperParam[Int]

     

165. class InternalWrapper extends Annotation with StaticAnnotation

     

     Generate the internal wrapper for a given class.

     Generate the internal wrapper for a given class. Used for complicated wrappers, where the basic functionality is auto-generated, and the rest is added in the inherited wrapper.

166. class JSONInputParser extends HTTPInputParser with HasURL with ComplexParamsWritable

     

167. class JSONOutputParser extends HTTPOutputParser with ComplexParamsWritable

     

     Annotations

     @InternalWrapper()

168. class KeyPhraseExtractor extends TextAnalyticsBase

     

169. case class KeyPhraseScore(id: String, keyPhrases: Seq[String]) extends Product with Serializable

     

170. trait LIMEBase extends LIMEParams with ComplexParamsWritable

     

171. trait LIMEParams extends HasInputCol with HasOutputCol with HasPredictionCol

     

172. class Lambda extends Transformer with Wrappable with ComplexParamsWritable

     

173. class LanguageDetector extends TextAnalyticsBase

     

174. trait LightGBMBase[TrainedModel <: Model[TrainedModel]] extends Estimator[TrainedModel] with LightGBMParams with org.apache.spark.ml.param.shared.HasFeaturesCol with org.apache.spark.ml.param.shared.HasLabelCol

     

175. class LightGBMBooster extends Serializable

     

     Represents a LightGBM Booster learner

     Represents a LightGBM Booster learner

     Annotations

     @SerialVersionUID()

176. class LightGBMClassificationModel extends ProbabilisticClassificationModel[Vector, LightGBMClassificationModel] with ConstructorWritable[LightGBMClassificationModel]

     

     Model produced by LightGBMClassifier.

     Model produced by LightGBMClassifier.

     Annotations

     @InternalWrapper()

177. class LightGBMClassifier extends ProbabilisticClassifier[Vector, LightGBMClassifier, LightGBMClassificationModel] with LightGBMBase[LightGBMClassificationModel]

     

     Trains a LightGBM Binary Classification model, a fast, distributed, high performance gradient boosting framework based on decision tree algorithms.

     Trains a LightGBM Binary Classification model, a fast, distributed, high performance gradient boosting framework based on decision tree algorithms. For more information please see here: https://github.com/Microsoft/LightGBM. For parameter information see here: https://github.com/Microsoft/LightGBM/blob/master/docs/Parameters.rst

     Annotations

     @InternalWrapper()

178. trait LightGBMParams extends Wrappable with DefaultParamsWritable with HasWeightCol with HasValidationIndicatorCol

     

     Defines common parameters across all LightGBM learners.

179. class LightGBMRegressionModel extends RegressionModel[Vector, LightGBMRegressionModel] with ConstructorWritable[LightGBMRegressionModel]

     

     Model produced by LightGBMRegressor.

     Model produced by LightGBMRegressor.

     Annotations

     @InternalWrapper()

180. class LightGBMRegressor extends BaseRegressor[Vector, LightGBMRegressor, LightGBMRegressionModel] with LightGBMBase[LightGBMRegressionModel]

     

     Trains a LightGBM Regression model, a fast, distributed, high performance gradient boosting framework based on decision tree algorithms.

     Trains a LightGBM Regression model, a fast, distributed, high performance gradient boosting framework based on decision tree algorithms. For more information please see here: https://github.com/Microsoft/LightGBM. For parameter information see here: https://github.com/Microsoft/LightGBM/blob/master/docs/Parameters.rst Note: The application parameter supports the following values:

     • regression_l2, L2 loss, alias=regression, mean_squared_error, mse, l2_root, root_mean_squared_error, rmse
     • regression_l1, L1 loss, alias=mean_absolute_error, mae
     • huber, Huber loss
     • fair, Fair loss
     • poisson, Poisson regression
     • quantile, Quantile regression
     • mape, MAPE loss, alias=mean_absolute_percentage_error
     • gamma, Gamma regression with log-link. It might be useful, e.g., for modeling insurance claims severity, or for any target that might be gamma-distributed
     • tweedie, Tweedie regression with log-link. It might be useful, e.g., for modeling total loss in insurance, or for any target that might be tweedie-distributed

     Annotations

     @InternalWrapper()

181. trait LinuxOnly extends TestBase

     

182. class LocalNER extends TextAnalyticsBase

     

183. case class LocalNEREntity(value: String, startIndex: Int, precision: Double, category: String) extends Product with Serializable

     

184. case class LocalNERScore(id: String, entities: Seq[LocalNEREntity]) extends Product with Serializable

     

185. class LocalWriter extends SingleFileResolver

     

186. class LongRangeHyperParam extends RangeHyperParam[Long]

     

187. class MMLConfig extends Configuration

     

188. class MPICommandBuilder extends CNTKCommandBuilderBase with MPIConfiguration

     

189. trait MPIConfiguration extends AnyRef

     

190. case class Match(text: String, offset: Int, length: Int) extends Product with Serializable

     

191. class MetricsLogger extends AnyRef

     

     Helper class for logging metrics to log4j.

192. trait MiniBatchBase extends Transformer with DefaultParamsWritable with Wrappable

     

193. trait Mode extends HasRecommenderCols

     

194. class ModelDownloader extends Client

     

     Class for downloading models from a server to Local or HDFS

195. class ModelNotFoundException extends FileNotFoundException

     

     Exception returned if a repo cannot find the file

196. case class ModelSchema(name: String, dataset: String, modelType: String, uri: URI, hash: String, size: Long, inputNode: Int, numLayers: Int, layerNames: Array[String]) extends Schema with Product with Serializable

     

     Class representing the schema of a CNTK model

     Class representing the schema of a CNTK model

     name

     name of the model architecture

     dataset

     dataset the model was trained on

     modelType

     type of problem the model is suited for eg: (image, text, sound, sentiment etc)

     uri

     location of the underlying file (local, HDFS, or HTTP)

     hash

     sha256 hash of the underlying file

     size

     size in bytes of the underlying file

     inputNode

     the node which represents the input

     numLayers

     the number of layers of the model

     layerNames

     the names nodes that represent layers in the network

197. class MultiColumnAdapter extends Estimator[PipelineModel] with Wrappable with ComplexParamsWritable

     

     The MultiColumnAdapter takes a unary pipeline stage and a list of input output column pairs and applies the pipeline stage to each input column after being fit

     The MultiColumnAdapter takes a unary pipeline stage and a list of input output column pairs and applies the pipeline stage to each input column after being fit

198. class MultiNGram extends Transformer with HasInputCol with HasOutputCol with Wrappable with DefaultParamsWritable

     

     Extracts several ngrams

199. class MultiVectorAssembler extends VectorAssembler

     

200. class NER extends TextAnalyticsBase

     

201. case class NERDoc(id: String, entities: Seq[NEREntity]) extends Product with Serializable

     

202. case class NEREntity(name: String, matches: Seq[NERMatch], type: Option[String], subtype: Option[String], wikipediaLanguage: Option[String], wikipediaId: Option[String], wikipediaUrl: Option[String], bingId: Option[String]) extends Product with Serializable

     

203. case class NERMatch(text: String, offset: Int, length: Int) extends Product with Serializable

     

204. class NativeLoader extends Serializable

     

205. case class NetworkParams(defaultListenPort: Int, addr: String, port: Int) extends Product with Serializable

     

206. class NullOrdering[T] extends Ordering[T]

     

207. class OCR extends CognitiveServicesBase with HasLanguage with HasImageInput with HasDetectOrientation with HasCognitiveServiceInput with HasInternalJsonOutputParser

     

208. class PageSplitter extends Transformer with HasInputCol with HasOutputCol with Wrappable with DefaultParamsWritable

     

     Splits text into chunks of at most n characters

209. class PartitionConsolidator extends Transformer with HTTPParams with HasInputCol with HasOutputCol with ComplexParamsWritable

     

210. sealed class PartitionSample extends Transformer with PartitionSampleParams

     

211. trait PartitionSampleParams extends Wrappable with DefaultParamsWritable

     

212. case class ProtocolVersionData(protocol: String, major: Int, minor: Int) extends Product with Serializable

     

213. class RandomDatasetGenerationConstraints extends HasDatasetGenerationConstraints

     

     Contraints on generating a dataset where all parameters are randomly generated.

214. abstract class RandomMMLGenerator[T] extends RandomDataGenerator[T]

     

     Base abstract class for random generation of data.

     Base abstract class for random generation of data.

     T

     The data to generate.

215. class RandomRowGenerator extends RandomMMLGenerator[Row]

     

     Randomly generates a row given the set space of data, column options.

216. class RandomRowGeneratorCombiner extends RandomMMLGenerator[Row]

     

     Combines an array of row generators into a single row generator.

217. class RandomSpace extends ParamSpace

     

     Represents a generator of parameters with specified distributions added by the HyperparamBuilder.

218. abstract class RangeHyperParam[T] extends Dist[T]

     

219. class RankingAdapter extends Estimator[RankingAdapterModel] with ComplexParamsWritable with RankingParams with Mode

     

220. class RankingAdapterModel extends Model[RankingAdapterModel] with ComplexParamsWritable with Wrappable with RankingParams with Mode

     

     Model from train validation split.

221. class RankingEvaluator extends Evaluator with RecEvaluatorParams with HasRecommenderCols with ComplexParamsWritable

     

222. trait RankingParams extends HasRecommenderCols with HasLabelCol with hasK

     

223. class RankingTrainValidationSplit extends Estimator[RankingTrainValidationSplitModel] with RankingTrainValidationSplitParams with Wrappable with ComplexParamsWritable with RecommendationParams

     

     Annotations

     @InternalWrapper()

224. class RankingTrainValidationSplitModel extends Model[RankingTrainValidationSplitModel] with Wrappable with ComplexParamsWritable

     

     Annotations

     @InternalWrapper()

225. class RecognizeDomainSpecificContent extends CognitiveServicesBase with HasImageInput with HasServiceParams with HasCognitiveServiceInput with HasInternalJsonOutputParser

     

226. class RecognizeText extends CognitiveServicesBaseWithoutHandler with HasImageInput with HasCognitiveServiceInput with HasInternalJsonOutputParser

     

227. class RecommendationIndexer extends Estimator[RecommendationIndexerModel] with RecommendationIndexerBase

     

228. trait RecommendationIndexerBase extends Params with ComplexParamsWritable

     

229. class RecommendationIndexerModel extends Model[RecommendationIndexerModel] with RecommendationIndexerBase with Wrappable

     

230. case class RegressorTrainParams(parallelism: String, numIterations: Int, learningRate: Double, numLeaves: Int, objective: String, alpha: Double, tweedieVariancePower: Double, maxBin: Int, baggingFraction: Double, baggingFreq: Int, baggingSeed: Int, earlyStoppingRound: Int, featureFraction: Double, maxDepth: Int, minSumHessianInLeaf: Double, numMachines: Int, modelString: Option[String], verbosity: Int, categoricalFeatures: Array[Int], boostFromAverage: Boolean, boostingType: String, lambdaL1: Double, lambdaL2: Double) extends TrainParams with Product with Serializable

     

     Defines the Booster parameters passed to the LightGBM regressor.

231. class RenameColumn extends Transformer with Wrappable with DefaultParamsWritable with HasInputCol with HasOutputCol

     

     RenameColumn takes a dataframe with an input and an output column name and returns a dataframe comprised of the original columns with the input column renamed as the output column name.

     RenameColumn takes a dataframe with an input and an output column name and returns a dataframe comprised of the original columns with the input column renamed as the output column name.

232. class Repartition extends Transformer with Wrappable with DefaultParamsWritable

     

     Partitions the dataset into n partitions

233. case class RequestLineData(method: String, uri: String, protoclVersion: Option[ProtocolVersionData]) extends Product with Serializable

     

234. class ResizeImage extends ImageTransformerStage

     

     Resizes the image.

     Resizes the image. The parameters of the ParameterMap are: "height" - the height of the image "width" "stageName" Please refer to OpenCV for more information

235. class ResizeImageTransformer extends Transformer with HasInputCol with HasOutputCol with Wrappable with DefaultParamsWritable

     

     Annotations

     @InternalWrapper()

236. class SAR extends Estimator[SARModel] with SARParams with DefaultParamsWritable

     

     Smart Adaptive Recommendations (SAR) Algorithm

     Smart Adaptive Recommendations (SAR) Algorithm

     https://aka.ms/reco-sar

     SAR is a fast scalable adaptive algorithm for personalized recommendations based on user transactions history and items description. It produces easily explainable / interpretable recommendations

     SAR has been show to provide higher ranking measurements when compared to ALS. https://github.com/Microsoft/Recommenders

     Annotations

     @InternalWrapper()

237. class SARModel extends Model[SARModel] with BaseRecommendationModel with Wrappable with SARParams with ComplexParamsWritable

     

     SAR Model

     SAR Model

     Annotations

     @InternalWrapper()

238. trait SARParams extends Wrappable with RecommendationParams

     

239. abstract class Schema extends AnyRef

     

     Abstract representation of a schema for an item that can be held in a repository

240. class SelectColumns extends Transformer with Wrappable with DefaultParamsWritable

     

     SelectColumns takes a dataframe and a list of columns to select as input and returns a dataframe comprised of only those columns listed in the input list.

     SelectColumns takes a dataframe and a list of columns to select as input and returns a dataframe comprised of only those columns listed in the input list.

     The columns to be selected is a list of column names

241. case class SentimentScore(id: String, score: Float) extends Product with Serializable

     

242. class SharedSingleton[T] extends Serializable

     

     Holds a variable shared among all workers that behaves like a local singleton.

     Holds a variable shared among all workers that behaves like a local singleton. Useful to use non-serializable objects in Spark closures that maintain state across tasks.

243. class SharedVariable[T] extends Serializable

     

     Holds a variable shared among all workers.

     Holds a variable shared among all workers. Useful to use non-serializable objects in Spark closures.

     Note this code has been borrowed from: https://www.nicolaferraro.me/2016/02/22/using-non-serializable-objects-in-apache-spark/

244. class SimpleHTTPTransformer extends Transformer with HTTPParams with HasMiniBatcher with HasHandler with HasInputCol with HasOutputCol with ComplexParamsWritable with HasErrorCol

     

     Annotations

     @InternalWrapper()

245. abstract class SingleFileResolver extends DataWriter

     

246. class SingleThreadedHTTPClient extends HTTPClient with SingleThreadedClient

     

247. abstract class SingleTypeReducer extends Transformer with TypeConversion

     

248. class SingleVectorAssembler extends VectorAssembler

     

249. case class StatusLineData(protocolVersion: ProtocolVersionData, statusCode: Int, reasonPhrase: String) extends Product with Serializable

     

250. class StringOutputParser extends HTTPOutputParser with ComplexParamsWritable

     

251. class SummarizeData extends Transformer with SummarizeDataParams

     

     Compute summary statistics for the dataset.

     Compute summary statistics for the dataset. The following statistics are computed: - counts - basic - sample - percentiles - errorThreshold - error threshold for quantiles

252. trait SummarizeDataParams extends Wrappable with DefaultParamsWritable

     

253. class Superpixel extends org.apache.spark.internal.Logging

     

254. case class SuperpixelData(clusters: Seq[Seq[(Int, Int)]]) extends Product with Serializable

     

255. class SuperpixelTransformer extends Transformer with HasInputCol with HasOutputCol with Wrappable with DefaultParamsWritable with HasCellSize with HasModifier

     

     A transformer that decomposes an image into it's superpixels

256. case class TADocument(language: Option[String], id: String, text: String) extends Product with Serializable

     

257. case class TAError(id: String, message: String) extends Product with Serializable

     

258. case class TARequest(documents: Seq[TADocument]) extends Product with Serializable

     

259. case class TAResponse[T](documents: Seq[T], errors: Option[Seq[TAError]]) extends Product with Serializable

     

260. class TLCConfig extends MMLConfig

     

261. class TabularLIME extends Estimator[TabularLIMEModel] with LIMEParams with Wrappable with ComplexParamsWritable

     

262. class TabularLIMEModel extends Model[TabularLIMEModel] with LIMEBase with Wrappable

     

263. class TagImage extends CognitiveServicesBase with HasImageInput with HasCognitiveServiceInput with HasInternalJsonOutputParser

     

264. abstract class TestBase extends FunSuite with BeforeAndAfterEachTestData with BeforeAndAfterAll

     

265. abstract class TextAnalyticsBase extends CognitiveServicesBase with HasCognitiveServiceInput with HasInternalJsonOutputParser

     

266. class TextFeaturizer extends Estimator[TextFeaturizerModel] with TextFeaturizerParams with HasInputCol with HasOutputCol

     

     Featurize text.

267. class TextFeaturizerModel extends Model[TextFeaturizerModel] with ConstructorWritable[TextFeaturizerModel]

     

268. trait TextFeaturizerParams extends Wrappable with DefaultParamsWritable

     

269. class TextPreprocessor extends Transformer with HasInputCol with HasOutputCol with Wrappable with ComplexParamsWritable

     

     TextPreprocessor takes a dataframe and a dictionary that maps (text -> replacement text), scans each cell in the input col and replaces all substring matches with the corresponding value.

     TextPreprocessor takes a dataframe and a dictionary that maps (text -> replacement text), scans each cell in the input col and replaces all substring matches with the corresponding value. Priority is given to longer keys and from left to right.

270. class TextSentiment extends TextAnalyticsBase

     

271. class Threshold extends ImageTransformerStage

     

     Applies a threshold to each element of the image.

     Applies a threshold to each element of the image. Please refer to threshold for more information

272. class TimeIntervalBatcher[T] extends Iterator[List[T]]

     

273. class TimeIntervalMiniBatchTransformer extends Transformer with MiniBatchBase

     

274. class Timer extends Estimator[TimerModel] with TimerParams with ComplexParamsWritable

     

275. class TimerModel extends Model[TimerModel] with TimerParams with ConstructorWritable[TimerModel]

     

276. trait TimerParams extends Wrappable

     

277. class TrainClassifier extends Estimator[TrainedClassifierModel] with AutoTrainer[TrainedClassifierModel]

     

     Trains a classification model.

     Trains a classification model. Featurizes the given data into a vector of doubles.

     Note the behavior of the reindex and labels parameters, the parameters interact as:

     reindex -> false labels -> false (Empty) Assume all double values, don't use metadata, assume natural ordering

     reindex -> true labels -> false (Empty) Index, use natural ordering of string indexer

     reindex -> false labels -> true (Specified) Assume user knows indexing, apply label values. Currently only string type supported.

     reindex -> true labels -> true (Specified) Validate labels matches column type, try to recast to label type, reindex label column

     The currently supported classifiers are: Logistic Regression Classifier Decision Tree Classifier Random Forest Classifier Gradient Boosted Trees Classifier Naive Bayes Classifier Multilayer Perceptron Classifier In addition to any generic learner that inherits from Predictor.

     Annotations

     @InternalWrapper()

278. abstract class TrainParams extends Serializable

     

     Defines the common Booster parameters passed to the LightGBM learners.

279. class TrainRegressor extends Estimator[TrainedRegressorModel] with AutoTrainer[TrainedRegressorModel]

     

     Trains a regression model.

     Trains a regression model.

     Annotations

     @InternalWrapper()

280. class TrainedClassifierModel extends AutoTrainedModel[TrainedClassifierModel]

     

     Model produced by TrainClassifier.

     Model produced by TrainClassifier.

     Annotations

     @InternalWrapper()

281. class TrainedRegressorModel extends AutoTrainedModel[TrainedRegressorModel]

     

     Model produced by TrainRegressor.

     Model produced by TrainRegressor.

     Annotations

     @InternalWrapper()

282. class Trie extends Serializable

     

283. class TuneHyperparameters extends Estimator[TuneHyperparametersModel] with Wrappable with ComplexParamsWritable with HasEvaluationMetric

     

     Tunes model hyperparameters

     Tunes model hyperparameters

     Allows user to specify multiple untrained models to tune using various search strategies. Currently supports cross validation with random grid search.

     Annotations

     @InternalWrapper()

284. class TuneHyperparametersModel extends Model[TuneHyperparametersModel] with ConstructorWritable[TuneHyperparametersModel]

     

     Model produced by TuneHyperparameters.

     Model produced by TuneHyperparameters.

     Annotations

     @InternalWrapper()

285. trait TypeConversion extends AnyRef

     

286. class UDFTransformer extends Transformer with Wrappable with ComplexParamsWritable with HasInputCol with HasInputCols with HasOutputCol

     

     UDFTransformer takes as input input column, output column, and a UserDefinedFunction returns a dataframe comprised of the original columns with the output column as the result of the udf applied to the input column

     UDFTransformer takes as input input column, output column, and a UserDefinedFunction returns a dataframe comprised of the original columns with the output column as the result of the udf applied to the input column

     Annotations

     @InternalWrapper()

287. class UnicodeNormalize extends Transformer with HasInputCol with HasOutputCol with Wrappable with ComplexParamsWritable

     

     UnicodeNormalize takes a dataframe and normalizes the unicode representation.

     UnicodeNormalize takes a dataframe and normalizes the unicode representation.

288. class UnrollBinaryImage extends Transformer with HasInputCol with HasOutputCol with Wrappable with DefaultParamsWritable

     

     Converts the representation of an m X n pixel image to an m * n vector of Doubles

     Converts the representation of an m X n pixel image to an m * n vector of Doubles

     The input column name is assumed to be "image", the output column name is "<uid>_output"

289. class UnrollImage extends Transformer with HasInputCol with HasOutputCol with Wrappable with DefaultParamsWritable

     

     Converts the representation of an m X n pixel image to an m * n vector of Doubles

     Converts the representation of an m X n pixel image to an m * n vector of Doubles

     The input column name is assumed to be "image", the output column name is "<uid>_output"

290. class ValueIndexer extends Estimator[ValueIndexerModel] with ValueIndexerParams

     

     Fits a dictionary of values from the input column.

     Fits a dictionary of values from the input column. Model then transforms a column to a categorical column of the given array of values. Similar to StringIndexer except it can be used on any value types.

291. class ValueIndexerModel extends Model[ValueIndexerModel] with ValueIndexerParams with ComplexParamsWritable

     

     Model produced by ValueIndexer.

292. trait ValueIndexerParams extends Wrappable with DefaultParamsWritable with HasInputCol with HasOutputCol

     

293. abstract class VectorAssembler extends AnyRef

     

294. class VerifyFaces extends CognitiveServicesBase with HasServiceParams with HasCognitiveServiceInput with HasInternalJsonOutputParser

     

295. trait Wrappable extends Params