class BatchDoFn[InputT] extends DoFn[InputT, Iterable[InputT]]
Batches input into a desired batch size.
Elements are buffered until there are enough elements for a batch, at which point they are emitted to the output PCollection
Windows are preserved (batches contain elements from the same window). Batches are not spanning over bundles. Once a bundle is finished, the batch is emitted even if not full. This function can only batch 10 parallel windows. If new element comes from an 11th window, the bigger batch will be emitted to give room for this new element.
- Source
- BatchDoFn.java
Linear Supertypes
Ordering
- Alphabetic
- By Inheritance
Inherited
- BatchDoFn
- DoFn
- HasDisplayData
- Serializable
- AnyRef
- Any
- Hide All
- Show All
Visibility
- Public
- Protected
Instance Constructors
- new BatchDoFn(maxWeight: Long, weigher: SerializableFunction[InputT, Long], maxLiveWindows: Int)
- new BatchDoFn(maxWeight: Long, weigher: SerializableFunction[InputT, Long])
Type Members
- abstract class FinishBundleContext extends AnyRef
- Definition Classes
- DoFn
- abstract class OnTimerContext extends WindowedContext
- Definition Classes
- DoFn
- abstract class OnWindowExpirationContext extends WindowedContext
- Definition Classes
- DoFn
- abstract class ProcessContext extends WindowedContext
- Definition Classes
- DoFn
- abstract class StartBundleContext extends AnyRef
- Definition Classes
- DoFn
- abstract class WindowedContext extends AnyRef
- Definition Classes
- DoFn
Abstract Value Members
Concrete Value Members
- final def !=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
- final def ##: Int
- Definition Classes
- AnyRef → Any
- final def ==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
- final def asInstanceOf[T0]: T0
- Definition Classes
- Any
- def clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.CloneNotSupportedException]) @native()
- final def eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
- def equals(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef → Any
- def finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.Throwable])
- def finishBundle(context: FinishBundleContext): Unit
- Annotations
- @FinishBundle()
- final def getClass(): Class[_ <: AnyRef]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
- def getInputTypeDescriptor(): TypeDescriptor[InputT]
- Definition Classes
- DoFn
- def getOutputTypeDescriptor(): TypeDescriptor[Iterable[InputT]]
- Definition Classes
- DoFn
- def hashCode(): Int
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
- final def isInstanceOf[T0]: Boolean
- Definition Classes
- Any
- final def ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
- final def notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
- final def notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
- def populateDisplayData(builder: Builder): Unit
- Definition Classes
- DoFn → HasDisplayData
- def processElement(element: InputT, window: BoundedWindow, out: OutputReceiver[Iterable[InputT]]): Unit
- Annotations
- @ProcessElement()
- def setup(): Unit
- Annotations
- @Setup()
- final def synchronized[T0](arg0: => T0): T0
- Definition Classes
- AnyRef
- def toString(): String
- Definition Classes
- AnyRef → Any
- final def wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException])
- final def wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException])
- final def wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws(classOf[java.lang.InterruptedException]) @native()