mirror of
https://github.com/open-metadata/OpenMetadata
synced 2026-05-24 09:39:11 +00:00
dfd2bd5167
* Fix: Airflow Task Instance link does not correctly filter by DAG * fix airflow pipeline source url
100 lines
No EOL
3.8 KiB
JSON
100 lines
No EOL
3.8 KiB
JSON
{
|
|
"tasks": [{
|
|
"name": "hive_create_table",
|
|
"displayName": "Hive Create Table",
|
|
"description": "Hive Create Table Task",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=hive_create_table",
|
|
"downstreamTasks": ["assert_table_exits"],
|
|
"taskType": "HiveOperator"
|
|
},
|
|
{
|
|
"name": "assert_table_exists",
|
|
"displayName": "Assert Table Exists",
|
|
"description": "Assert if a table exists",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=assert_table_exists",
|
|
"downstreamTasks": [],
|
|
"taskType": "HiveOperator"
|
|
},
|
|
{
|
|
"name": "snowflake_task",
|
|
"displayName": "Snowflake Task",
|
|
"description": "Airflow operator to perform ETL on snowflake tables",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=assert_table_exists",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "SnowflakeOperator"
|
|
},
|
|
{
|
|
"name": "presto_task",
|
|
"displayName": "Presto Task",
|
|
"description": "Airflow operator to perform ETL on presto tables",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=assert_table_exists",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "PrestoOperator"
|
|
},
|
|
{
|
|
"name": "dim_address_task",
|
|
"displayName": "dim_address Task",
|
|
"description": "Airflow operator to perform ETL and generate dim_address table",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=dim_address_task",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "PrestoOperator"
|
|
},
|
|
{
|
|
"name": "dim_user_task",
|
|
"displayName": "dim_user Task",
|
|
"description": "Airflow operator to perform ETL and generate dim_user table",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=dim_user_task",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "PrestoOperator"
|
|
},
|
|
{
|
|
"name": "dim_location_task",
|
|
"displayName": "dim_location Task",
|
|
"description": "Airflow operator to perform ETL and generate dim_location table",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=dim_location_task",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "PrestoOperator"
|
|
},
|
|
{
|
|
"name": "trino_task",
|
|
"displayName": "Trino Task",
|
|
"description": "Airflow operator to perform ETL on trino tables",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=assert_table_exists",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "TrinoOperator"
|
|
},
|
|
{
|
|
"name": "dim_location_etl",
|
|
"displayName": "dim_location etl",
|
|
"description": "dim_location ETL pipeline",
|
|
"sourceUrl": "http://localhost:8080/tree?dag_id=dim_address_etl",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=assert_table_exists",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "TrinoOperator"
|
|
},
|
|
{
|
|
"name": "metrics_aggregation_task",
|
|
"displayName": "Metrics Aggregation Task",
|
|
"description": "Airflow operator to aggregate streaming metrics in real-time",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=metrics_aggregation_task",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "PythonOperator"
|
|
},
|
|
{
|
|
"name": "feature_engineering_task",
|
|
"displayName": "Feature Engineering Task",
|
|
"description": "Airflow operator to generate ML features from warehouse data",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=feature_engineering_task",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "PythonOperator"
|
|
},
|
|
{
|
|
"name": "dim_product_task",
|
|
"displayName": "dim_product Task",
|
|
"description": "Airflow operator to perform ETL and generate dim_product table",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?_flt_3_dag_id=dim_product_task",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "PrestoOperator"
|
|
}
|
|
]
|
|
} |