Package io.openlineage.client

Class OpenLineage

java.lang.Object
io.openlineage.client.OpenLineage
public final class OpenLineage extends Object
Usage: 

   URI producer = URI.create("http://my.producer/uri");
   OpenLineage ol = new OpenLineage(producer);
   UUID runId = UUID.randomUUID();
   RunFacets runFacets =
     ol.newRunFacetsBuilder().nominalTime(ol.newNominalTimeRunFacet(now, now)).build();
   Run run = ol.newRun(runId, runFacets);
   String name = "jobName";
   String namespace = "namespace";
   JobFacets jobFacets = ol.newJobFacetsBuilder().build();
   Job job = ol.newJob(namespace, name, jobFacets);
   List<InputDataset> inputs = Arrays.asList(ol.newInputDataset("ins", "input", null, null));
   List<OutputDataset> outputs = Arrays.asList(ol.newOutputDataset("ons", "output", null, null));
   RunEvent runStateUpdate =
     ol.newRunEvent(now, OpenLineage.RunEvent.EventType.START, run, job, inputs, outputs);

  • Constructor Details

    • OpenLineage

      public OpenLineage(URI producer)
      Starting point to create OpenLineage objects. Use the OpenLineage instance to create events and facets
      Parameters:
      producer - the identifier of the library using the client to generate OpenLineage events

  • Method Details

    • newSchemaDatasetFacetFields

      public OpenLineage.SchemaDatasetFacetFields newSchemaDatasetFacetFields(String name, String type, String description, List<OpenLineage.SchemaDatasetFacetFields> fields)
      Factory method for SchemaDatasetFacetFields
      Parameters:
      name - The name of the field.
      type - The type of the field.
      description - The description of the field.
      fields - Nested struct fields.
      Returns:
      SchemaDatasetFacetFields

    • newSchemaDatasetFacetFieldsBuilder

      public OpenLineage.SchemaDatasetFacetFieldsBuilder newSchemaDatasetFacetFieldsBuilder()
      Creates a builder for SchemaDatasetFacetFields
      Returns:
      a new builder for SchemaDatasetFacetFields

    • newParentRunFacet

      public OpenLineage.ParentRunFacet newParentRunFacet(OpenLineage.ParentRunFacetRun run, OpenLineage.ParentRunFacetJob job, OpenLineage.ParentRunFacetRoot root)
      Factory method for ParentRunFacet
      Parameters:
      run - the run
      job - the job
      root - the root
      Returns:
      ParentRunFacet

    • newParentRunFacetBuilder

      public OpenLineage.ParentRunFacetBuilder newParentRunFacetBuilder()
      Creates a builder for ParentRunFacet
      Returns:
      a new builder for ParentRunFacet

    • newOutputStatisticsOutputDatasetFacet

      public OpenLineage.OutputStatisticsOutputDatasetFacet newOutputStatisticsOutputDatasetFacet(Long rowCount, Long size, Long fileCount)
      Factory method for OutputStatisticsOutputDatasetFacet
      Parameters:
      rowCount - The number of rows written to the dataset
      size - The size in bytes written to the dataset
      fileCount - The number of files written to the dataset
      Returns:
      OutputStatisticsOutputDatasetFacet

    • newOutputStatisticsOutputDatasetFacetBuilder

      public OpenLineage.OutputStatisticsOutputDatasetFacetBuilder newOutputStatisticsOutputDatasetFacetBuilder()
      Creates a builder for OutputStatisticsOutputDatasetFacet
      Returns:
      a new builder for OutputStatisticsOutputDatasetFacet

    • newExtractionErrorRunFacetErrors

      public OpenLineage.ExtractionErrorRunFacetErrors newExtractionErrorRunFacetErrors(String errorMessage, String stackTrace, String task, Long taskNumber)
      Factory method for ExtractionErrorRunFacetErrors
      Parameters:
      errorMessage - Text representation of extraction error message.
      stackTrace - Stack trace of extraction error message
      task - Text representation of task that failed. This can be, for example, SQL statement that parser could not interpret.
      taskNumber - Order of task (counted from 0).
      Returns:
      ExtractionErrorRunFacetErrors

    • newExtractionErrorRunFacetErrorsBuilder

      public OpenLineage.ExtractionErrorRunFacetErrorsBuilder newExtractionErrorRunFacetErrorsBuilder()
      Creates a builder for ExtractionErrorRunFacetErrors
      Returns:
      a new builder for ExtractionErrorRunFacetErrors

    • newJobTypeJobFacet

      public OpenLineage.JobTypeJobFacet newJobTypeJobFacet(String processingType, String integration, String jobType)
      Factory method for JobTypeJobFacet
      Parameters:
      processingType - Job processing type like: BATCH or STREAMING
      integration - OpenLineage integration type of this job: for example SPARK|DBT|AIRFLOW|FLINK
      jobType - Run type, for example: QUERY|COMMAND|DAG|TASK|JOB|MODEL. This is an integration-specific field.
      Returns:
      JobTypeJobFacet

    • newJobTypeJobFacetBuilder

      public OpenLineage.JobTypeJobFacetBuilder newJobTypeJobFacetBuilder()
      Creates a builder for JobTypeJobFacet
      Returns:
      a new builder for JobTypeJobFacet

    • newRun

      public OpenLineage.Run newRun(UUID runId, OpenLineage.RunFacets facets)
      Factory method for Run
      Parameters:
      runId - The globally unique ID of the run associated with the job.
      facets - The run facets.
      Returns:
      Run

    • newRunBuilder

      public OpenLineage.RunBuilder newRunBuilder()
      Creates a builder for Run
      Returns:
      a new builder for Run

    • newParentRunFacetRoot

      public OpenLineage.ParentRunFacetRoot newParentRunFacetRoot(OpenLineage.RootRun run, OpenLineage.RootJob job)
      Factory method for ParentRunFacetRoot
      Parameters:
      run - the run
      job - the job
      Returns:
      ParentRunFacetRoot

    • newParentRunFacetRootBuilder

      public OpenLineage.ParentRunFacetRootBuilder newParentRunFacetRootBuilder()
      Creates a builder for ParentRunFacetRoot
      Returns:
      a new builder for ParentRunFacetRoot

    • newExternalQueryRunFacet

      public OpenLineage.ExternalQueryRunFacet newExternalQueryRunFacet(String externalQueryId, String source)
      Factory method for ExternalQueryRunFacet
      Parameters:
      externalQueryId - Identifier for the external system
      source - source of the external query
      Returns:
      ExternalQueryRunFacet

    • newExternalQueryRunFacetBuilder

      public OpenLineage.ExternalQueryRunFacetBuilder newExternalQueryRunFacetBuilder()
      Creates a builder for ExternalQueryRunFacet
      Returns:
      a new builder for ExternalQueryRunFacet

    • newDataQualityMetricsInputDatasetFacetColumnMetricsAdditionalQuantiles

      public OpenLineage.DataQualityMetricsInputDatasetFacetColumnMetricsAdditionalQuantiles newDataQualityMetricsInputDatasetFacetColumnMetricsAdditionalQuantiles()
      Factory method for DataQualityMetricsInputDatasetFacetColumnMetricsAdditionalQuantiles
      Returns:
      DataQualityMetricsInputDatasetFacetColumnMetricsAdditionalQuantiles

    • newDataQualityMetricsInputDatasetFacetColumnMetricsAdditionalQuantilesBuilder

      public OpenLineage.DataQualityMetricsInputDatasetFacetColumnMetricsAdditionalQuantilesBuilder newDataQualityMetricsInputDatasetFacetColumnMetricsAdditionalQuantilesBuilder()
      Creates a builder for DataQualityMetricsInputDatasetFacetColumnMetricsAdditionalQuantiles
      Returns:
      a new builder for DataQualityMetricsInputDatasetFacetColumnMetricsAdditionalQuantiles

    • newOutputDatasetOutputFacets

      public OpenLineage.OutputDatasetOutputFacets newOutputDatasetOutputFacets(OpenLineage.IcebergCommitReportOutputDatasetFacet iceberg_scan_report, OpenLineage.OutputStatisticsOutputDatasetFacet outputStatistics)
      Factory method for OutputDatasetOutputFacets
      Parameters:
      iceberg_scan_report - the iceberg_scan_report
      outputStatistics - the outputStatistics
      Returns:
      OutputDatasetOutputFacets

    • newOutputDatasetOutputFacetsBuilder

      public OpenLineage.OutputDatasetOutputFacetsBuilder newOutputDatasetOutputFacetsBuilder()
      Creates a builder for OutputDatasetOutputFacets
      Returns:
      a new builder for OutputDatasetOutputFacets

    • newNominalTimeRunFacet

      public OpenLineage.NominalTimeRunFacet newNominalTimeRunFacet(ZonedDateTime nominalStartTime, ZonedDateTime nominalEndTime)
      Factory method for NominalTimeRunFacet
      Parameters:
      nominalStartTime - An [ISO-8601](https://en.wikipedia.org/wiki/ISO_8601) timestamp representing the nominal start time (included) of the run. AKA the schedule time
      nominalEndTime - An [ISO-8601](https://en.wikipedia.org/wiki/ISO_8601) timestamp representing the nominal end time (excluded) of the run. (Should be the nominal start time of the next run)
      Returns:
      NominalTimeRunFacet

    • newNominalTimeRunFacetBuilder

      public OpenLineage.NominalTimeRunFacetBuilder newNominalTimeRunFacetBuilder()
      Creates a builder for NominalTimeRunFacet
      Returns:
      a new builder for NominalTimeRunFacet

    • newInputDatasetFacet

      public OpenLineage.InputDatasetFacet newInputDatasetFacet()
      Returns:
      InputDatasetFacet

    • newColumnLineageDatasetFacetFieldsAdditional

      public OpenLineage.ColumnLineageDatasetFacetFieldsAdditional newColumnLineageDatasetFacetFieldsAdditional(List<OpenLineage.InputField> inputFields, String transformationDescription, String transformationType)
      Factory method for ColumnLineageDatasetFacetFieldsAdditional
      Parameters:
      inputFields - the inputFields
      transformationDescription - a string representation of the transformation applied
      transformationType - IDENTITY|MASKED reflects a clearly defined behavior. IDENTITY: exact same as input; MASKED: no original data available (like a hash of PII for example)
      Returns:
      ColumnLineageDatasetFacetFieldsAdditional

    • newColumnLineageDatasetFacetFieldsAdditionalBuilder

      public OpenLineage.ColumnLineageDatasetFacetFieldsAdditionalBuilder newColumnLineageDatasetFacetFieldsAdditionalBuilder()
      Creates a builder for ColumnLineageDatasetFacetFieldsAdditional
      Returns:
      a new builder for ColumnLineageDatasetFacetFieldsAdditional

    • newDataQualityMetricsInputDatasetFacetColumnMetricsAdditional

      public OpenLineage.DataQualityMetricsInputDatasetFacetColumnMetricsAdditional newDataQualityMetricsInputDatasetFacetColumnMetricsAdditional(Long nullCount, Long distinctCount, Double sum, Double count, Double min, Double max, OpenLineage.DataQualityMetricsInputDatasetFacetColumnMetricsAdditionalQuantiles quantiles)
      Factory method for DataQualityMetricsInputDatasetFacetColumnMetricsAdditional
      Parameters:
      nullCount - The number of null values in this column for the rows evaluated
      distinctCount - The number of distinct values in this column for the rows evaluated
      sum - The total sum of values in this column for the rows evaluated
      count - The number of values in this column
      min - the min
      max - the max
      quantiles - The property key is the quantile. Examples: 0.1 0.25 0.5 0.75 1
      Returns:
      DataQualityMetricsInputDatasetFacetColumnMetricsAdditional

    • newDataQualityMetricsInputDatasetFacetColumnMetricsAdditionalBuilder

      public OpenLineage.DataQualityMetricsInputDatasetFacetColumnMetricsAdditionalBuilder newDataQualityMetricsInputDatasetFacetColumnMetricsAdditionalBuilder()
      Creates a builder for DataQualityMetricsInputDatasetFacetColumnMetricsAdditional
      Returns:
      a new builder for DataQualityMetricsInputDatasetFacetColumnMetricsAdditional

    • newDataQualityMetricsInputDatasetFacet

      public OpenLineage.DataQualityMetricsInputDatasetFacet newDataQualityMetricsInputDatasetFacet(Long rowCount, Long bytes, Long fileCount, OpenLineage.DataQualityMetricsInputDatasetFacetColumnMetrics columnMetrics)
      Factory method for DataQualityMetricsInputDatasetFacet
      Parameters:
      rowCount - The number of rows evaluated
      bytes - The size in bytes
      fileCount - The number of files evaluated
      columnMetrics - The property key is the column name
      Returns:
      DataQualityMetricsInputDatasetFacet

    • newDataQualityMetricsInputDatasetFacetBuilder

      public OpenLineage.DataQualityMetricsInputDatasetFacetBuilder newDataQualityMetricsInputDatasetFacetBuilder()
      Creates a builder for DataQualityMetricsInputDatasetFacet
      Returns:
      a new builder for DataQualityMetricsInputDatasetFacet

    • newDocumentationJobFacet

      public OpenLineage.DocumentationJobFacet newDocumentationJobFacet(String description)
      Factory method for DocumentationJobFacet
      Parameters:
      description - The description of the job.
      Returns:
      DocumentationJobFacet

    • newDocumentationJobFacetBuilder

      public OpenLineage.DocumentationJobFacetBuilder newDocumentationJobFacetBuilder()
      Creates a builder for DocumentationJobFacet
      Returns:
      a new builder for DocumentationJobFacet

    • newInputDatasetInputFacets

      public OpenLineage.InputDatasetInputFacets newInputDatasetInputFacets(OpenLineage.DataQualityAssertionsDatasetFacet dataQualityAssertions, OpenLineage.InputStatisticsInputDatasetFacet inputStatistics, OpenLineage.DataQualityMetricsInputDatasetFacet dataQualityMetrics, OpenLineage.IcebergScanReportInputDatasetFacet iceberg_scan_report)
      Factory method for InputDatasetInputFacets
      Parameters:
      dataQualityAssertions - the dataQualityAssertions
      inputStatistics - the inputStatistics
      dataQualityMetrics - the dataQualityMetrics
      iceberg_scan_report - the iceberg_scan_report
      Returns:
      InputDatasetInputFacets

    • newInputDatasetInputFacetsBuilder

      public OpenLineage.InputDatasetInputFacetsBuilder newInputDatasetInputFacetsBuilder()
      Creates a builder for InputDatasetInputFacets
      Returns:
      a new builder for InputDatasetInputFacets

    • newSQLJobFacet

      public OpenLineage.SQLJobFacet newSQLJobFacet(String query)
      Factory method for SQLJobFacet
      Parameters:
      query - the query
      Returns:
      SQLJobFacet

    • newSQLJobFacetBuilder

      public OpenLineage.SQLJobFacetBuilder newSQLJobFacetBuilder()
      Creates a builder for SQLJobFacet
      Returns:
      a new builder for SQLJobFacet

    • newOutputDataset

      public OpenLineage.OutputDataset newOutputDataset(String namespace, String name, OpenLineage.DatasetFacets facets, OpenLineage.OutputDatasetOutputFacets outputFacets)
      Factory method for OutputDataset
      Parameters:
      namespace - The namespace containing that dataset
      name - The unique name for that dataset within that namespace
      facets - The facets for this dataset
      outputFacets - The output facets for this dataset
      Returns:
      OutputDataset

    • newOutputDatasetBuilder

      public OpenLineage.OutputDatasetBuilder newOutputDatasetBuilder()
      Creates a builder for OutputDataset
      Returns:
      a new builder for OutputDataset

    • newTagsDatasetFacetFields

      public OpenLineage.TagsDatasetFacetFields newTagsDatasetFacetFields(String key, String value, String source, String field)
      Factory method for TagsDatasetFacetFields
      Parameters:
      key - Key that identifies the tag
      value - The value of the field
      source - The source of the tag. INTEGRATION|USER|DBT CORE|SPARK|etc.
      field - Identifies the field in a dataset if a tag applies to one
      Returns:
      TagsDatasetFacetFields

    • newTagsDatasetFacetFieldsBuilder

      public OpenLineage.TagsDatasetFacetFieldsBuilder newTagsDatasetFacetFieldsBuilder()
      Creates a builder for TagsDatasetFacetFields
      Returns:
      a new builder for TagsDatasetFacetFields

    • newRootJob

      public OpenLineage.RootJob newRootJob(String namespace, String name)
      Factory method for RootJob
      Parameters:
      namespace - The namespace containing root job
      name - The unique name containing root job within that namespace
      Returns:
      RootJob

    • newRootJobBuilder

      public OpenLineage.RootJobBuilder newRootJobBuilder()
      Creates a builder for RootJob
      Returns:
      a new builder for RootJob

    • newDocumentationDatasetFacet

      public OpenLineage.DocumentationDatasetFacet newDocumentationDatasetFacet(String description)
      Factory method for DocumentationDatasetFacet
      Parameters:
      description - The description of the dataset.
      Returns:
      DocumentationDatasetFacet

    • newDocumentationDatasetFacetBuilder

      public OpenLineage.DocumentationDatasetFacetBuilder newDocumentationDatasetFacetBuilder()
      Creates a builder for DocumentationDatasetFacet
      Returns:
      a new builder for DocumentationDatasetFacet

    • newSourceCodeLocationJobFacet

      public OpenLineage.SourceCodeLocationJobFacet newSourceCodeLocationJobFacet(String type, URI url, String repoUrl, String path, String version, String tag, String branch)
      Factory method for SourceCodeLocationJobFacet
      Parameters:
      type - the source control system
      url - the full http URL to locate the file
      repoUrl - the URL to the repository
      path - the path in the repo containing the source files
      version - the current version deployed (not a branch name, the actual unique version)
      tag - optional tag name
      branch - optional branch name
      Returns:
      SourceCodeLocationJobFacet

    • newSourceCodeLocationJobFacetBuilder

      public OpenLineage.SourceCodeLocationJobFacetBuilder newSourceCodeLocationJobFacetBuilder()
      Creates a builder for SourceCodeLocationJobFacet
      Returns:
      a new builder for SourceCodeLocationJobFacet

    • newIcebergScanReportInputDatasetFacet

      public OpenLineage.IcebergScanReportInputDatasetFacet newIcebergScanReportInputDatasetFacet(Double snapshotId, String filter, Double schemaId, List<String> projectedFieldNames, OpenLineage.IcebergScanReportInputDatasetFacetScanMetrics scanMetrics, OpenLineage.IcebergScanReportInputDatasetFacetMetadata metadata)
      Factory method for IcebergScanReportInputDatasetFacet
      Parameters:
      snapshotId - Snapshot ID of the iceberg table
      filter - Filter used to scan the iceberg table
      schemaId - Schema ID of the iceberg table
      projectedFieldNames - List of field names that are projected from the iceberg table
      scanMetrics - the scanMetrics
      metadata - the metadata
      Returns:
      IcebergScanReportInputDatasetFacet

    • newIcebergScanReportInputDatasetFacetBuilder

      public OpenLineage.IcebergScanReportInputDatasetFacetBuilder newIcebergScanReportInputDatasetFacetBuilder()
      Creates a builder for IcebergScanReportInputDatasetFacet
      Returns:
      a new builder for IcebergScanReportInputDatasetFacet

    • newDatasetTypeDatasetFacet

      public OpenLineage.DatasetTypeDatasetFacet newDatasetTypeDatasetFacet(String datasetType, String subType)
      Factory method for DatasetTypeDatasetFacet
      Parameters:
      datasetType - Dataset type, for example: FILE|STREAM|MODEL|TABLE|VIEW.
      subType - Optional sub-type within the dataset type (e.g., BEST VALIDATION under MODEL CHECKPOINT).
      Returns:
      DatasetTypeDatasetFacet

    • newDatasetTypeDatasetFacetBuilder

      public OpenLineage.DatasetTypeDatasetFacetBuilder newDatasetTypeDatasetFacetBuilder()
      Creates a builder for DatasetTypeDatasetFacet
      Returns:
      a new builder for DatasetTypeDatasetFacet

    • newJob

      public OpenLineage.Job newJob(String namespace, String name, OpenLineage.JobFacets facets)
      Factory method for Job
      Parameters:
      namespace - The namespace containing that job
      name - The unique name for that job within that namespace
      facets - The job facets.
      Returns:
      Job

    • newJobBuilder

      public OpenLineage.JobBuilder newJobBuilder()
      Creates a builder for Job
      Returns:
      a new builder for Job

    • newDatasetVersionDatasetFacet

      public OpenLineage.DatasetVersionDatasetFacet newDatasetVersionDatasetFacet(String datasetVersion)
      Factory method for DatasetVersionDatasetFacet
      Parameters:
      datasetVersion - The version of the dataset.
      Returns:
      DatasetVersionDatasetFacet

    • newDatasetVersionDatasetFacetBuilder

      public OpenLineage.DatasetVersionDatasetFacetBuilder newDatasetVersionDatasetFacetBuilder()
      Creates a builder for DatasetVersionDatasetFacet
      Returns:
      a new builder for DatasetVersionDatasetFacet

    • newSymlinksDatasetFacetIdentifiers

      public OpenLineage.SymlinksDatasetFacetIdentifiers newSymlinksDatasetFacetIdentifiers(String namespace, String name, String type)
      Factory method for SymlinksDatasetFacetIdentifiers
      Parameters:
      namespace - The dataset namespace
      name - The dataset name
      type - Identifier type
      Returns:
      SymlinksDatasetFacetIdentifiers

    • newSymlinksDatasetFacetIdentifiersBuilder

      public OpenLineage.SymlinksDatasetFacetIdentifiersBuilder newSymlinksDatasetFacetIdentifiersBuilder()
      Creates a builder for SymlinksDatasetFacetIdentifiers
      Returns:
      a new builder for SymlinksDatasetFacetIdentifiers

    • newIcebergCommitReportOutputDatasetFacet

      public OpenLineage.IcebergCommitReportOutputDatasetFacet newIcebergCommitReportOutputDatasetFacet(Double snapshotId, Double sequenceNumber, String operation, OpenLineage.IcebergCommitReportOutputDatasetFacetCommitMetrics commitMetrics, OpenLineage.IcebergCommitReportOutputDatasetFacetMetadata metadata)
      Factory method for IcebergCommitReportOutputDatasetFacet
      Parameters:
      snapshotId - Snapshot ID of the iceberg table
      sequenceNumber - Sequence number of the iceberg table
      operation - Operation that was performed on the iceberg table
      commitMetrics - the commitMetrics
      metadata - the metadata
      Returns:
      IcebergCommitReportOutputDatasetFacet

    • newIcebergCommitReportOutputDatasetFacetBuilder

      public OpenLineage.IcebergCommitReportOutputDatasetFacetBuilder newIcebergCommitReportOutputDatasetFacetBuilder()
      Creates a builder for IcebergCommitReportOutputDatasetFacet
      Returns:
      a new builder for IcebergCommitReportOutputDatasetFacet

    • newTagsJobFacetFields

      public OpenLineage.TagsJobFacetFields newTagsJobFacetFields(String key, String value, String source)
      Factory method for TagsJobFacetFields
      Parameters:
      key - Key that identifies the tag
      value - The value of the field
      source - The source of the tag. INTEGRATION|USER|DBT CORE|SPARK|etc.
      Returns:
      TagsJobFacetFields

    • newTagsJobFacetFieldsBuilder

      public OpenLineage.TagsJobFacetFieldsBuilder newTagsJobFacetFieldsBuilder()
      Creates a builder for TagsJobFacetFields
      Returns:
      a new builder for TagsJobFacetFields

    • newOutputDatasetFacet

      public OpenLineage.OutputDatasetFacet newOutputDatasetFacet()
      Returns:
      OutputDatasetFacet

    • newStaticDataset

      public OpenLineage.StaticDataset newStaticDataset(String namespace, String name, OpenLineage.DatasetFacets facets)
      Factory method for StaticDataset
      Parameters:
      namespace - The namespace containing that dataset
      name - The unique name for that dataset within that namespace
      facets - The facets for this dataset
      Returns:
      StaticDataset

    • newStaticDatasetBuilder

      public OpenLineage.StaticDatasetBuilder newStaticDatasetBuilder()
      Creates a builder for StaticDataset
      Returns:
      a new builder for StaticDataset

    • newTagsRunFacet

      public OpenLineage.TagsRunFacet newTagsRunFacet(List<OpenLineage.TagsRunFacetFields> tags)
      Factory method for TagsRunFacet
      Parameters:
      tags - The tags applied to the run facet
      Returns:
      TagsRunFacet

    • newTagsRunFacetBuilder

      public OpenLineage.TagsRunFacetBuilder newTagsRunFacetBuilder()
      Creates a builder for TagsRunFacet
      Returns:
      a new builder for TagsRunFacet

    • newLifecycleStateChangeDatasetFacet

      public OpenLineage.LifecycleStateChangeDatasetFacet newLifecycleStateChangeDatasetFacet(OpenLineage.LifecycleStateChangeDatasetFacet.LifecycleStateChange lifecycleStateChange, OpenLineage.LifecycleStateChangeDatasetFacetPreviousIdentifier previousIdentifier)
      Factory method for LifecycleStateChangeDatasetFacet
      Parameters:
      lifecycleStateChange - The lifecycle state change.
      previousIdentifier - Previous name of the dataset in case of renaming it.
      Returns:
      LifecycleStateChangeDatasetFacet

    • newLifecycleStateChangeDatasetFacetBuilder

      public OpenLineage.LifecycleStateChangeDatasetFacetBuilder newLifecycleStateChangeDatasetFacetBuilder()
      Creates a builder for LifecycleStateChangeDatasetFacet
      Returns:
      a new builder for LifecycleStateChangeDatasetFacet

    • newRunEvent

      public OpenLineage.RunEvent newRunEvent(ZonedDateTime eventTime, OpenLineage.RunEvent.EventType eventType, OpenLineage.Run run, OpenLineage.Job job, List<OpenLineage.InputDataset> inputs, List<OpenLineage.OutputDataset> outputs)
      Factory method for RunEvent
      Parameters:
      eventTime - the time the event occurred at
      eventType - the current transition of the run state. It is required to issue 1 START event and 1 of [ COMPLETE, ABORT, FAIL ] event per run. Additional events with OTHER eventType can be added to the same run. For example to send additional metadata after the run is complete
      run - the run
      job - the job
      inputs - The set of **input** datasets.
      outputs - The set of **output** datasets.
      Returns:
      RunEvent

    • newRunEventBuilder

      public OpenLineage.RunEventBuilder newRunEventBuilder()
      Creates a builder for RunEvent
      Returns:
      a new builder for RunEvent

    • newIcebergCommitReportOutputDatasetFacetMetadata

      public OpenLineage.IcebergCommitReportOutputDatasetFacetMetadata newIcebergCommitReportOutputDatasetFacetMetadata()
      Factory method for IcebergCommitReportOutputDatasetFacetMetadata
      Returns:
      IcebergCommitReportOutputDatasetFacetMetadata

    • newIcebergCommitReportOutputDatasetFacetMetadataBuilder

      public OpenLineage.IcebergCommitReportOutputDatasetFacetMetadataBuilder newIcebergCommitReportOutputDatasetFacetMetadataBuilder()
      Creates a builder for IcebergCommitReportOutputDatasetFacetMetadata
      Returns:
      a new builder for IcebergCommitReportOutputDatasetFacetMetadata

    • newTagsDatasetFacet

      public OpenLineage.TagsDatasetFacet newTagsDatasetFacet(List<OpenLineage.TagsDatasetFacetFields> tags)
      Factory method for TagsDatasetFacet
      Parameters:
      tags - The tags applied to the dataset facet
      Returns:
      TagsDatasetFacet

    • newTagsDatasetFacetBuilder

      public OpenLineage.TagsDatasetFacetBuilder newTagsDatasetFacetBuilder()
      Creates a builder for TagsDatasetFacet
      Returns:
      a new builder for TagsDatasetFacet

    • newRootRun

      public OpenLineage.RootRun newRootRun(UUID runId)
      Factory method for RootRun
      Parameters:
      runId - The globally unique ID of the root run associated with the root job.
      Returns:
      RootRun

    • newRootRunBuilder

      public OpenLineage.RootRunBuilder newRootRunBuilder()
      Creates a builder for RootRun
      Returns:
      a new builder for RootRun

    • newRunFacet

      public OpenLineage.RunFacet newRunFacet()
      Returns:
      RunFacet

    • newOwnershipJobFacetOwners

      public OpenLineage.OwnershipJobFacetOwners newOwnershipJobFacetOwners(String name, String type)
      Factory method for OwnershipJobFacetOwners
      Parameters:
      name - the identifier of the owner of the Job. It is recommended to define this as a URN. For example application:foo, user:jdoe, team:data
      type - The type of ownership (optional)
      Returns:
      OwnershipJobFacetOwners

    • newOwnershipJobFacetOwnersBuilder

      public OpenLineage.OwnershipJobFacetOwnersBuilder newOwnershipJobFacetOwnersBuilder()
      Creates a builder for OwnershipJobFacetOwners
      Returns:
      a new builder for OwnershipJobFacetOwners

    • newGcpLineageJobFacet

      public OpenLineage.GcpLineageJobFacet newGcpLineageJobFacet(String displayName, OpenLineage.GcpLineageJobFacetOrigin origin)
      Factory method for GcpLineageJobFacet
      Parameters:
      displayName - The name of the job to be used on UI
      origin - the origin
      Returns:
      GcpLineageJobFacet

    • newGcpLineageJobFacetBuilder

      public OpenLineage.GcpLineageJobFacetBuilder newGcpLineageJobFacetBuilder()
      Creates a builder for GcpLineageJobFacet
      Returns:
      a new builder for GcpLineageJobFacet

    • newJobEvent

      public OpenLineage.JobEvent newJobEvent(ZonedDateTime eventTime, OpenLineage.Job job, List<OpenLineage.InputDataset> inputs, List<OpenLineage.OutputDataset> outputs)
      Factory method for JobEvent
      Parameters:
      eventTime - the time the event occurred at
      job - the job
      inputs - The set of **input** datasets.
      outputs - The set of **output** datasets.
      Returns:
      JobEvent

    • newJobEventBuilder

      public OpenLineage.JobEventBuilder newJobEventBuilder()
      Creates a builder for JobEvent
      Returns:
      a new builder for JobEvent

    • newOwnershipDatasetFacetOwners

      public OpenLineage.OwnershipDatasetFacetOwners newOwnershipDatasetFacetOwners(String name, String type)
      Factory method for OwnershipDatasetFacetOwners
      Parameters:
      name - the identifier of the owner of the Dataset. It is recommended to define this as a URN. For example application:foo, user:jdoe, team:data
      type - The type of ownership (optional)
      Returns:
      OwnershipDatasetFacetOwners

    • newOwnershipDatasetFacetOwnersBuilder

      public OpenLineage.OwnershipDatasetFacetOwnersBuilder newOwnershipDatasetFacetOwnersBuilder()
      Creates a builder for OwnershipDatasetFacetOwners
      Returns:
      a new builder for OwnershipDatasetFacetOwners

    • newExtractionErrorRunFacet

      public OpenLineage.ExtractionErrorRunFacet newExtractionErrorRunFacet(Long totalTasks, Long failedTasks, List<OpenLineage.ExtractionErrorRunFacetErrors> errors)
      Factory method for ExtractionErrorRunFacet
      Parameters:
      totalTasks - The number of distinguishable tasks in a run that were processed by OpenLineage, whether successfully or not. Those could be, for example, distinct SQL statements.
      failedTasks - The number of distinguishable tasks in a run that were processed not successfully by OpenLineage. Those could be, for example, distinct SQL statements.
      errors - the errors
      Returns:
      ExtractionErrorRunFacet

    • newExtractionErrorRunFacetBuilder

      public OpenLineage.ExtractionErrorRunFacetBuilder newExtractionErrorRunFacetBuilder()
      Creates a builder for ExtractionErrorRunFacet
      Returns:
      a new builder for ExtractionErrorRunFacet

    • newDataQualityMetricsInputDatasetFacetColumnMetrics

      public OpenLineage.DataQualityMetricsInputDatasetFacetColumnMetrics newDataQualityMetricsInputDatasetFacetColumnMetrics()
      Factory method for DataQualityMetricsInputDatasetFacetColumnMetrics
      Returns:
      DataQualityMetricsInputDatasetFacetColumnMetrics

    • newDataQualityMetricsInputDatasetFacetColumnMetricsBuilder

      public OpenLineage.DataQualityMetricsInputDatasetFacetColumnMetricsBuilder newDataQualityMetricsInputDatasetFacetColumnMetricsBuilder()
      Creates a builder for DataQualityMetricsInputDatasetFacetColumnMetrics
      Returns:
      a new builder for DataQualityMetricsInputDatasetFacetColumnMetrics

    • newInputField

      public OpenLineage.InputField newInputField(String namespace, String name, String field, List<OpenLineage.InputFieldTransformations> transformations)
      Factory method for InputField
      Parameters:
      namespace - The input dataset namespace
      name - The input dataset name
      field - The input field
      transformations - the transformations
      Returns:
      InputField

    • newInputFieldBuilder

      public OpenLineage.InputFieldBuilder newInputFieldBuilder()
      Creates a builder for InputField
      Returns:
      a new builder for InputField

    • newInputStatisticsInputDatasetFacet

      public OpenLineage.InputStatisticsInputDatasetFacet newInputStatisticsInputDatasetFacet(Long rowCount, Long size, Long fileCount)
      Factory method for InputStatisticsInputDatasetFacet
      Parameters:
      rowCount - The number of rows read
      size - The size in bytes read
      fileCount - The number of files read
      Returns:
      InputStatisticsInputDatasetFacet

    • newInputStatisticsInputDatasetFacetBuilder

      public OpenLineage.InputStatisticsInputDatasetFacetBuilder newInputStatisticsInputDatasetFacetBuilder()
      Creates a builder for InputStatisticsInputDatasetFacet
      Returns:
      a new builder for InputStatisticsInputDatasetFacet

    • newJobFacets

      public OpenLineage.JobFacets newJobFacets(OpenLineage.JobTypeJobFacet jobType, OpenLineage.SourceCodeJobFacet sourceCode, OpenLineage.GcpLineageJobFacet gcp_lineage, OpenLineage.SQLJobFacet sql, OpenLineage.OwnershipJobFacet ownership, OpenLineage.SourceCodeLocationJobFacet sourceCodeLocation, OpenLineage.TagsJobFacet tags, OpenLineage.DocumentationJobFacet documentation)
      Factory method for JobFacets
      Parameters:
      jobType - the jobType
      sourceCode - the sourceCode
      gcp_lineage - the gcp_lineage
      sql - the sql
      ownership - the ownership
      sourceCodeLocation - the sourceCodeLocation
      tags - the tags
      documentation - the documentation
      Returns:
      JobFacets

    • newJobFacetsBuilder

      public OpenLineage.JobFacetsBuilder newJobFacetsBuilder()
      Creates a builder for JobFacets
      Returns:
      a new builder for JobFacets

    • newIcebergCommitReportOutputDatasetFacetCommitMetrics

      public OpenLineage.IcebergCommitReportOutputDatasetFacetCommitMetrics newIcebergCommitReportOutputDatasetFacetCommitMetrics(Double totalDuration, Double attempts, Double addedDataFiles, Double removedDataFiles, Double totalDataFiles, Double addedDeleteFiles, Double addedEqualityDeleteFiles, Double addedPositionalDeleteFiles, Double addedDVs, Double removedDeleteFiles, Double removedEqualityDeleteFiles, Double removedPositionalDeleteFiles, Double removedDVs, Double totalDeleteFiles, Double addedRecords, Double removedRecords, Double totalRecords, Double addedFilesSizeInBytes, Double removedFilesSizeInBytes, Double totalFilesSizeInBytes, Double addedPositionalDeletes, Double removedPositionalDeletes, Double totalPositionalDeletes, Double addedEqualityDeletes, Double removedEqualityDeletes, Double totalEqualityDeletes)
      Factory method for IcebergCommitReportOutputDatasetFacetCommitMetrics
      Parameters:
      totalDuration - Duration of the commit in MILLISECONDS
      attempts - Number of attempts made to commit the iceberg table
      addedDataFiles - Number of data files that are added during the commit
      removedDataFiles - Number of data files that are removed during the commit
      totalDataFiles - Total number of data files that are present in the iceberg table
      addedDeleteFiles - Number of delete files that are added during the commit
      addedEqualityDeleteFiles - Number of added equality delete files
      addedPositionalDeleteFiles - Number of added positional delete files
      addedDVs - Number of added DVs
      removedDeleteFiles - Number of delete files that are removed during the commit
      removedEqualityDeleteFiles - Number of removed equality delete files
      removedPositionalDeleteFiles - Number of removed positional delete files
      removedDVs - Number of removed DVs
      totalDeleteFiles - Total number of temporary delete files that are present in the iceberg table
      addedRecords - Number of records that are added during the commit
      removedRecords - Number of records that are removed during the commit
      totalRecords - Number of records that are present in the iceberg table
      addedFilesSizeInBytes - Number of files size in bytes that are added during the commit
      removedFilesSizeInBytes - Number of files size in bytes that are removed during the commit
      totalFilesSizeInBytes - Number of files size in bytes in the iceberg table
      addedPositionalDeletes - Number of positional deletes that are added during the commit
      removedPositionalDeletes - Number of positional deletes that are removed during the commit
      totalPositionalDeletes - Number of positional deletes that are present in the iceberg table
      addedEqualityDeletes - Number of equality deletes that are added during the commit
      removedEqualityDeletes - Number of equality deletes that are removed during the commit
      totalEqualityDeletes - Number of equality deletes that are present in the iceberg table
      Returns:
      IcebergCommitReportOutputDatasetFacetCommitMetrics

    • newIcebergCommitReportOutputDatasetFacetCommitMetricsBuilder

      public OpenLineage.IcebergCommitReportOutputDatasetFacetCommitMetricsBuilder newIcebergCommitReportOutputDatasetFacetCommitMetricsBuilder()
      Creates a builder for IcebergCommitReportOutputDatasetFacetCommitMetrics
      Returns:
      a new builder for IcebergCommitReportOutputDatasetFacetCommitMetrics

    • newDatasetFacet

      public OpenLineage.DatasetFacet newDatasetFacet()
      Returns:
      DatasetFacet

    • newDeletedDatasetFacet

      public OpenLineage.DatasetFacet newDeletedDatasetFacet()
      Returns:
      a deleted DatasetFacet

    • newOwnershipDatasetFacet

      public OpenLineage.OwnershipDatasetFacet newOwnershipDatasetFacet(List<OpenLineage.OwnershipDatasetFacetOwners> owners)
      Factory method for OwnershipDatasetFacet
      Parameters:
      owners - The owners of the dataset.
      Returns:
      OwnershipDatasetFacet

    • newOwnershipDatasetFacetBuilder

      public OpenLineage.OwnershipDatasetFacetBuilder newOwnershipDatasetFacetBuilder()
      Creates a builder for OwnershipDatasetFacet
      Returns:
      a new builder for OwnershipDatasetFacet

    • newEnvironmentVariablesRunFacet

      public OpenLineage.EnvironmentVariablesRunFacet newEnvironmentVariablesRunFacet(List<OpenLineage.EnvironmentVariable> environmentVariables)
      Factory method for EnvironmentVariablesRunFacet
      Parameters:
      environmentVariables - The environment variables for the run.
      Returns:
      EnvironmentVariablesRunFacet

    • newEnvironmentVariablesRunFacetBuilder

      public OpenLineage.EnvironmentVariablesRunFacetBuilder newEnvironmentVariablesRunFacetBuilder()
      Creates a builder for EnvironmentVariablesRunFacet
      Returns:
      a new builder for EnvironmentVariablesRunFacet

    • newDatasetFacets

      public OpenLineage.DatasetFacets newDatasetFacets(OpenLineage.DatasourceDatasetFacet dataSource, OpenLineage.DatasetVersionDatasetFacet version, OpenLineage.DatasetTypeDatasetFacet datasetType, OpenLineage.StorageDatasetFacet storage, OpenLineage.ColumnLineageDatasetFacet columnLineage, OpenLineage.LifecycleStateChangeDatasetFacet lifecycleStateChange, OpenLineage.TagsDatasetFacet tags, OpenLineage.DocumentationDatasetFacet documentation, OpenLineage.SchemaDatasetFacet schema, OpenLineage.OwnershipDatasetFacet ownership, OpenLineage.SymlinksDatasetFacet symlinks)
      Factory method for DatasetFacets
      Parameters:
      dataSource - the dataSource
      version - the version
      datasetType - the datasetType
      storage - the storage
      columnLineage - the columnLineage
      lifecycleStateChange - the lifecycleStateChange
      tags - the tags
      documentation - the documentation
      schema - the schema
      ownership - the ownership
      symlinks - the symlinks
      Returns:
      DatasetFacets

    • newDatasetFacetsBuilder

      public OpenLineage.DatasetFacetsBuilder newDatasetFacetsBuilder()
      Creates a builder for DatasetFacets
      Returns:
      a new builder for DatasetFacets

    • newOwnershipJobFacet

      public OpenLineage.OwnershipJobFacet newOwnershipJobFacet(List<OpenLineage.OwnershipJobFacetOwners> owners)
      Factory method for OwnershipJobFacet
      Parameters:
      owners - The owners of the job.
      Returns:
      OwnershipJobFacet

    • newOwnershipJobFacetBuilder

      public OpenLineage.OwnershipJobFacetBuilder newOwnershipJobFacetBuilder()
      Creates a builder for OwnershipJobFacet
      Returns:
      a new builder for OwnershipJobFacet

    • newParentRunFacetJob

      public OpenLineage.ParentRunFacetJob newParentRunFacetJob(String namespace, String name)
      Factory method for ParentRunFacetJob
      Parameters:
      namespace - The namespace containing that job
      name - The unique name for that job within that namespace
      Returns:
      ParentRunFacetJob

    • newParentRunFacetJobBuilder

      public OpenLineage.ParentRunFacetJobBuilder newParentRunFacetJobBuilder()
      Creates a builder for ParentRunFacetJob
      Returns:
      a new builder for ParentRunFacetJob

    • newGcpLineageJobFacetOrigin

      public OpenLineage.GcpLineageJobFacetOrigin newGcpLineageJobFacetOrigin(String sourceType, String name)
      Factory method for GcpLineageJobFacetOrigin
      Parameters:
      sourceType - Type of the source. Possible values can be found in GCP documentation (https://cloud.google.com/data-catalog/docs/reference/data-lineage/rest/v1/projects.locations.processes#SourceType)
      name - If the sourceType isn't CUSTOM, the value of this field should be a GCP resource name of the system, which reports lineage. The project and location parts of the resource name must match the project and location of the lineage resource being created. More details in GCP documentation https://cloud.google.com/data-catalog/docs/reference/data-lineage/rest/v1/projects.locations.processes#origin
      Returns:
      GcpLineageJobFacetOrigin

    • newGcpLineageJobFacetOriginBuilder

      public OpenLineage.GcpLineageJobFacetOriginBuilder newGcpLineageJobFacetOriginBuilder()
      Creates a builder for GcpLineageJobFacetOrigin
      Returns:
      a new builder for GcpLineageJobFacetOrigin

    • newErrorMessageRunFacet

      public OpenLineage.ErrorMessageRunFacet newErrorMessageRunFacet(String message, String programmingLanguage, String stackTrace)
      Factory method for ErrorMessageRunFacet
      Parameters:
      message - A human-readable string representing error message generated by observed system
      programmingLanguage - Programming language the observed system uses.
      stackTrace - A language-specific stack trace generated by observed system
      Returns:
      ErrorMessageRunFacet

    • newErrorMessageRunFacetBuilder

      public OpenLineage.ErrorMessageRunFacetBuilder newErrorMessageRunFacetBuilder()
      Creates a builder for ErrorMessageRunFacet
      Returns:
      a new builder for ErrorMessageRunFacet

    • newJobFacet

      public OpenLineage.JobFacet newJobFacet()
      Returns:
      JobFacet

    • newDeletedJobFacet

      public OpenLineage.JobFacet newDeletedJobFacet()
      Returns:
      a deleted JobFacet

    • newRunFacets

      public OpenLineage.RunFacets newRunFacets(OpenLineage.ExternalQueryRunFacet externalQuery, OpenLineage.GcpDataprocRunFacet gcp_dataproc, OpenLineage.ExtractionErrorRunFacet extractionError, OpenLineage.ParentRunFacet parent, OpenLineage.NominalTimeRunFacet nominalTime, OpenLineage.TagsRunFacet tags, OpenLineage.ErrorMessageRunFacet errorMessage, OpenLineage.EnvironmentVariablesRunFacet environmentVariables, OpenLineage.ProcessingEngineRunFacet processing_engine)
      Factory method for RunFacets
      Parameters:
      externalQuery - the externalQuery
      gcp_dataproc - the gcp_dataproc
      extractionError - the extractionError
      parent - the parent
      nominalTime - the nominalTime
      tags - the tags
      errorMessage - the errorMessage
      environmentVariables - the environmentVariables
      processing_engine - the processing_engine
      Returns:
      RunFacets

    • newRunFacetsBuilder

      public OpenLineage.RunFacetsBuilder newRunFacetsBuilder()
      Creates a builder for RunFacets
      Returns:
      a new builder for RunFacets

    • newSchemaDatasetFacet

      public OpenLineage.SchemaDatasetFacet newSchemaDatasetFacet(List<OpenLineage.SchemaDatasetFacetFields> fields)
      Factory method for SchemaDatasetFacet
      Parameters:
      fields - The fields of the data source.
      Returns:
      SchemaDatasetFacet

    • newSchemaDatasetFacetBuilder

      public OpenLineage.SchemaDatasetFacetBuilder newSchemaDatasetFacetBuilder()
      Creates a builder for SchemaDatasetFacet
      Returns:
      a new builder for SchemaDatasetFacet

    • newSourceCodeJobFacet

      public OpenLineage.SourceCodeJobFacet newSourceCodeJobFacet(String language, String sourceCode)
      Factory method for SourceCodeJobFacet
      Parameters:
      language - Language in which source code of this job was written.
      sourceCode - Source code of this job.
      Returns:
      SourceCodeJobFacet

    • newSourceCodeJobFacetBuilder

      public OpenLineage.SourceCodeJobFacetBuilder newSourceCodeJobFacetBuilder()
      Creates a builder for SourceCodeJobFacet
      Returns:
      a new builder for SourceCodeJobFacet

    • newIcebergScanReportInputDatasetFacetMetadata

      public OpenLineage.IcebergScanReportInputDatasetFacetMetadata newIcebergScanReportInputDatasetFacetMetadata()
      Factory method for IcebergScanReportInputDatasetFacetMetadata
      Returns:
      IcebergScanReportInputDatasetFacetMetadata

    • newIcebergScanReportInputDatasetFacetMetadataBuilder

      public OpenLineage.IcebergScanReportInputDatasetFacetMetadataBuilder newIcebergScanReportInputDatasetFacetMetadataBuilder()
      Creates a builder for IcebergScanReportInputDatasetFacetMetadata
      Returns:
      a new builder for IcebergScanReportInputDatasetFacetMetadata

    • newColumnLineageDatasetFacet

      public OpenLineage.ColumnLineageDatasetFacet newColumnLineageDatasetFacet(OpenLineage.ColumnLineageDatasetFacetFields fields, List<OpenLineage.InputField> dataset)
      Factory method for ColumnLineageDatasetFacet
      Parameters:
      fields - Column level lineage that maps output fields into input fields used to evaluate them.
      dataset - Column level lineage that affects the whole dataset. This includes filtering, sorting, grouping (aggregates), joining, window functions, etc.
      Returns:
      ColumnLineageDatasetFacet

    • newColumnLineageDatasetFacetBuilder

      public OpenLineage.ColumnLineageDatasetFacetBuilder newColumnLineageDatasetFacetBuilder()
      Creates a builder for ColumnLineageDatasetFacet
      Returns:
      a new builder for ColumnLineageDatasetFacet

    • newDataQualityAssertionsDatasetFacet

      public OpenLineage.DataQualityAssertionsDatasetFacet newDataQualityAssertionsDatasetFacet(List<OpenLineage.DataQualityAssertionsDatasetFacetAssertions> assertions)
      Factory method for DataQualityAssertionsDatasetFacet
      Parameters:
      assertions - the assertions
      Returns:
      DataQualityAssertionsDatasetFacet

    • newDataQualityAssertionsDatasetFacetBuilder

      public OpenLineage.DataQualityAssertionsDatasetFacetBuilder newDataQualityAssertionsDatasetFacetBuilder()
      Creates a builder for DataQualityAssertionsDatasetFacet
      Returns:
      a new builder for DataQualityAssertionsDatasetFacet

    • newDatasetEvent

      public OpenLineage.DatasetEvent newDatasetEvent(ZonedDateTime eventTime, OpenLineage.StaticDataset dataset)
      Factory method for DatasetEvent
      Parameters:
      eventTime - the time the event occurred at
      dataset - the dataset
      Returns:
      DatasetEvent

    • newDatasetEventBuilder

      public OpenLineage.DatasetEventBuilder newDatasetEventBuilder()
      Creates a builder for DatasetEvent
      Returns:
      a new builder for DatasetEvent

    • newTagsJobFacet

      public OpenLineage.TagsJobFacet newTagsJobFacet(List<OpenLineage.TagsJobFacetFields> tags)
      Factory method for TagsJobFacet
      Parameters:
      tags - The tags applied to the job facet
      Returns:
      TagsJobFacet

    • newTagsJobFacetBuilder

      public OpenLineage.TagsJobFacetBuilder newTagsJobFacetBuilder()
      Creates a builder for TagsJobFacet
      Returns:
      a new builder for TagsJobFacet

    • newProcessingEngineRunFacet

      public OpenLineage.ProcessingEngineRunFacet newProcessingEngineRunFacet(String version, String name, String openlineageAdapterVersion)
      Factory method for ProcessingEngineRunFacet
      Parameters:
      version - Processing engine version. Might be Airflow or Spark version.
      name - Processing engine name, e.g. Airflow or Spark
      openlineageAdapterVersion - OpenLineage adapter package version. Might be e.g. OpenLineage Airflow integration package version
      Returns:
      ProcessingEngineRunFacet

    • newProcessingEngineRunFacetBuilder

      public OpenLineage.ProcessingEngineRunFacetBuilder newProcessingEngineRunFacetBuilder()
      Creates a builder for ProcessingEngineRunFacet
      Returns:
      a new builder for ProcessingEngineRunFacet

    • newLifecycleStateChangeDatasetFacetPreviousIdentifier

      public OpenLineage.LifecycleStateChangeDatasetFacetPreviousIdentifier newLifecycleStateChangeDatasetFacetPreviousIdentifier(String name, String namespace)
      Factory method for LifecycleStateChangeDatasetFacetPreviousIdentifier
      Parameters:
      name - the name
      namespace - the namespace
      Returns:
      LifecycleStateChangeDatasetFacetPreviousIdentifier

    • newLifecycleStateChangeDatasetFacetPreviousIdentifierBuilder

      public OpenLineage.LifecycleStateChangeDatasetFacetPreviousIdentifierBuilder newLifecycleStateChangeDatasetFacetPreviousIdentifierBuilder()
      Creates a builder for LifecycleStateChangeDatasetFacetPreviousIdentifier
      Returns:
      a new builder for LifecycleStateChangeDatasetFacetPreviousIdentifier

    • newInputFieldTransformations

      public OpenLineage.InputFieldTransformations newInputFieldTransformations(String type, String subtype, String description, Boolean masking)
      Factory method for InputFieldTransformations
      Parameters:
      type - The type of the transformation. Allowed values are: DIRECT, INDIRECT
      subtype - The subtype of the transformation
      description - a string representation of the transformation applied
      masking - is transformation masking the data or not
      Returns:
      InputFieldTransformations

    • newInputFieldTransformationsBuilder

      public OpenLineage.InputFieldTransformationsBuilder newInputFieldTransformationsBuilder()
      Creates a builder for InputFieldTransformations
      Returns:
      a new builder for InputFieldTransformations

    • newEnvironmentVariable

      public OpenLineage.EnvironmentVariable newEnvironmentVariable(String name, String value)
      Factory method for EnvironmentVariable
      Parameters:
      name - The name of the environment variable.
      value - The value of the environment variable.
      Returns:
      EnvironmentVariable

    • newEnvironmentVariableBuilder

      public OpenLineage.EnvironmentVariableBuilder newEnvironmentVariableBuilder()
      Creates a builder for EnvironmentVariable
      Returns:
      a new builder for EnvironmentVariable

    • newIcebergScanReportInputDatasetFacetScanMetrics

      public OpenLineage.IcebergScanReportInputDatasetFacetScanMetrics newIcebergScanReportInputDatasetFacetScanMetrics(Double totalPlanningDuration, Double resultDataFiles, Double resultDeleteFiles, Double totalDataManifests, Double totalDeleteManifests, Double scannedDataManifests, Double skippedDataManifests, Double totalFileSizeInBytes, Double totalDeleteFileSizeInBytes, Double skippedDataFiles, Double skippedDeleteFiles, Double scannedDeleteManifests, Double skippedDeleteManifests, Double indexedDeleteFiles, Double equalityDeleteFiles, Double positionalDeleteFiles)
      Factory method for IcebergScanReportInputDatasetFacetScanMetrics
      Parameters:
      totalPlanningDuration - Duration of the scan in MILLISECONDS
      resultDataFiles - List of data files that are read during the scan
      resultDeleteFiles - List of delete files that are read during the scan
      totalDataManifests - Total number of manifests that are scanned during the scan
      totalDeleteManifests - Total number of delete manifests that are scanned during the scan
      scannedDataManifests - Number of data manifests that are scanned during the scan
      skippedDataManifests - Number of data manifests that are skipped during the scan
      totalFileSizeInBytes - Total file size in bytes that are read during the scan
      totalDeleteFileSizeInBytes - Total delete file size in bytes that are read during the scan
      skippedDataFiles - Number of data files that are skipped during the scan
      skippedDeleteFiles - Number of delete files that are skipped during the scan
      scannedDeleteManifests - Number of delete manifests that are scanned during the scan
      skippedDeleteManifests - Number of delete manifests that are skipped during the scan
      indexedDeleteFiles - Number of delete files that are indexed during the scan
      equalityDeleteFiles - Number of delete files that are equality indexed during the scan
      positionalDeleteFiles - Number of delete files that are positional indexed during the scan
      Returns:
      IcebergScanReportInputDatasetFacetScanMetrics

    • newIcebergScanReportInputDatasetFacetScanMetricsBuilder

      public OpenLineage.IcebergScanReportInputDatasetFacetScanMetricsBuilder newIcebergScanReportInputDatasetFacetScanMetricsBuilder()
      Creates a builder for IcebergScanReportInputDatasetFacetScanMetrics
      Returns:
      a new builder for IcebergScanReportInputDatasetFacetScanMetrics

    • newTagsRunFacetFields

      public OpenLineage.TagsRunFacetFields newTagsRunFacetFields(String key, String value, String source)
      Factory method for TagsRunFacetFields
      Parameters:
      key - Key that identifies the tag
      value - The value of the field
      source - The source of the tag. INTEGRATION|USER|DBT CORE|SPARK|etc.
      Returns:
      TagsRunFacetFields

    • newTagsRunFacetFieldsBuilder

      public OpenLineage.TagsRunFacetFieldsBuilder newTagsRunFacetFieldsBuilder()
      Creates a builder for TagsRunFacetFields
      Returns:
      a new builder for TagsRunFacetFields

    • newDataQualityAssertionsDatasetFacetAssertions

      public OpenLineage.DataQualityAssertionsDatasetFacetAssertions newDataQualityAssertionsDatasetFacetAssertions(String assertion, Boolean success, String column)
      Factory method for DataQualityAssertionsDatasetFacetAssertions
      Parameters:
      assertion - Type of expectation test that dataset is subjected to
      success - the success
      column - Column that expectation is testing. It should match the name provided in SchemaDatasetFacet. If column field is empty, then expectation refers to whole dataset.
      Returns:
      DataQualityAssertionsDatasetFacetAssertions

    • newDataQualityAssertionsDatasetFacetAssertionsBuilder

      public OpenLineage.DataQualityAssertionsDatasetFacetAssertionsBuilder newDataQualityAssertionsDatasetFacetAssertionsBuilder()
      Creates a builder for DataQualityAssertionsDatasetFacetAssertions
      Returns:
      a new builder for DataQualityAssertionsDatasetFacetAssertions

    • newParentRunFacetRun

      public OpenLineage.ParentRunFacetRun newParentRunFacetRun(UUID runId)
      Factory method for ParentRunFacetRun
      Parameters:
      runId - The globally unique ID of the run associated with the job.
      Returns:
      ParentRunFacetRun

    • newParentRunFacetRunBuilder

      public OpenLineage.ParentRunFacetRunBuilder newParentRunFacetRunBuilder()
      Creates a builder for ParentRunFacetRun
      Returns:
      a new builder for ParentRunFacetRun

    • newGcpDataprocRunFacet

      public OpenLineage.GcpDataprocRunFacet newGcpDataprocRunFacet(String appId, String appName, String batchId, String batchUuid, String clusterName, String clusterUuid, String jobId, String jobUuid, String projectId, String queryNodeName, String jobType, String sessionId, String sessionUuid)
      Factory method for GcpDataprocRunFacet
      Parameters:
      appId - Application ID set by the resource manager. For spark jobs, it is set in the spark configuration of the current context.
      appName - App name which may be provided by the user, or some default is used by the resource manager. For spark jobs, it is set in the spark configuration of the current context.
      batchId - Populated only for Dataproc serverless batches. The resource id of the batch.
      batchUuid - Populated only for Dataproc serverless batches. A UUID generated by the service when it creates the batch.
      clusterName - Populated only for Dataproc GCE workloads. The cluster name is unique within a GCP project.
      clusterUuid - Populated only for Dataproc GCE workloads. A UUID generated by the service at the time of cluster creation.
      jobId - Populated only for Dataproc GCE workloads. If not specified by the user, the job ID will be provided by the service.
      jobUuid - Populated only for Dataproc GCE workloads. A UUID that uniquely identifies a job within the project over time.
      projectId - The GCP project ID that the resource belongs to.
      queryNodeName - The name of the query node in the executed Spark Plan. Often used to describe the command being executed.
      jobType - Identifies whether the process is a job (on a Dataproc cluster), a batch or a session.
      sessionId - Populated only for Dataproc serverless interactive sessions. The resource id of the session, used for URL generation.
      sessionUuid - Populated only for Dataproc serverless interactive sessions. A UUID generated by the service when it creates the session.
      Returns:
      GcpDataprocRunFacet

    • newGcpDataprocRunFacetBuilder

      public OpenLineage.GcpDataprocRunFacetBuilder newGcpDataprocRunFacetBuilder()
      Creates a builder for GcpDataprocRunFacet
      Returns:
      a new builder for GcpDataprocRunFacet

    • newColumnLineageDatasetFacetFields

      public OpenLineage.ColumnLineageDatasetFacetFields newColumnLineageDatasetFacetFields()
      Factory method for ColumnLineageDatasetFacetFields
      Returns:
      ColumnLineageDatasetFacetFields

    • newColumnLineageDatasetFacetFieldsBuilder

      public OpenLineage.ColumnLineageDatasetFacetFieldsBuilder newColumnLineageDatasetFacetFieldsBuilder()
      Creates a builder for ColumnLineageDatasetFacetFields
      Returns:
      a new builder for ColumnLineageDatasetFacetFields

    • newSymlinksDatasetFacet

      public OpenLineage.SymlinksDatasetFacet newSymlinksDatasetFacet(List<OpenLineage.SymlinksDatasetFacetIdentifiers> identifiers)
      Factory method for SymlinksDatasetFacet
      Parameters:
      identifiers - the identifiers
      Returns:
      SymlinksDatasetFacet

    • newSymlinksDatasetFacetBuilder

      public OpenLineage.SymlinksDatasetFacetBuilder newSymlinksDatasetFacetBuilder()
      Creates a builder for SymlinksDatasetFacet
      Returns:
      a new builder for SymlinksDatasetFacet

    • newStorageDatasetFacet

      public OpenLineage.StorageDatasetFacet newStorageDatasetFacet(String storageLayer, String fileFormat)
      Factory method for StorageDatasetFacet
      Parameters:
      storageLayer - Storage layer provider with allowed values: iceberg, delta.
      fileFormat - File format with allowed values: parquet, orc, avro, json, csv, text, xml.
      Returns:
      StorageDatasetFacet

    • newStorageDatasetFacetBuilder

      public OpenLineage.StorageDatasetFacetBuilder newStorageDatasetFacetBuilder()
      Creates a builder for StorageDatasetFacet
      Returns:
      a new builder for StorageDatasetFacet

    • newInputDataset

      public OpenLineage.InputDataset newInputDataset(String namespace, String name, OpenLineage.DatasetFacets facets, OpenLineage.InputDatasetInputFacets inputFacets)
      Factory method for InputDataset
      Parameters:
      namespace - The namespace containing that dataset
      name - The unique name for that dataset within that namespace
      facets - The facets for this dataset
      inputFacets - The input facets for this dataset.
      Returns:
      InputDataset

    • newInputDatasetBuilder

      public OpenLineage.InputDatasetBuilder newInputDatasetBuilder()
      Creates a builder for InputDataset
      Returns:
      a new builder for InputDataset

    • newDatasourceDatasetFacet

      public OpenLineage.DatasourceDatasetFacet newDatasourceDatasetFacet(String name, URI uri)
      Factory method for DatasourceDatasetFacet
      Parameters:
      name - the name
      uri - the uri
      Returns:
      DatasourceDatasetFacet

    • newDatasourceDatasetFacetBuilder

      public OpenLineage.DatasourceDatasetFacetBuilder newDatasourceDatasetFacetBuilder()
      Creates a builder for DatasourceDatasetFacet
      Returns:
      a new builder for DatasourceDatasetFacet