Iceberg

Scio supports reading from and writing to Apache Iceberg via Beam’s Managed transforms. Magnolify’s RowType (available as part of the magnolify-beam artifact) provides automatically-derived mappings between scala case classes and Beam’s Row, used by the underlying managed transform. See full documentation here.

To read:

import com.spotify.scio.ScioContext
import com.spotify.scio.iceberg._
import com.spotify.scio.values.SCollection
import magnolify.beam._

case class Record(a: Int, b: String)
implicit val rt: RowType[Record] = RowType[Record]

val sc: ScioContext = ???
val table: String = ???
val catalogName: String = ???
val catalogConfig: Map[String, String] = ???

val records: SCollection[Record] = sc.iceberg[Record](
  table, 
  catalogName, 
  catalogConfig
)

To write:

import com.spotify.scio.iceberg._
import com.spotify.scio.values.SCollection
import magnolify.beam._
case class Record(a: Int, b: String)
implicit val rt: RowType[Record] = RowType[Record]

val records: SCollection[Record] = ???

val table: String = ???
val catalogName: String = ???
val catalogConfig: Map[String, String] = ???

records.saveAsIceberg(table, catalogName, catalogConfig)