mirror of
https://github.com/open-metadata/OpenMetadata
synced 2026-05-24 09:39:11 +00:00
743ff85ee4
* init pipeline-profiler cli command * Fixed issues with the bulk sink * Update generated TypeScript types * fix and remove unnecessary code blocks * fix and remove unnecessary code blocks * Added get observ data by pipeline id api * Added APIs for metrics and charts * remove fallback mechanism * Build fixes * mvn build fixes * Api remove unnecssary changes * Fix Metrics API * Fix trends API * Fixed filtering * Added sample data * Added more sample data * Move to metadata workflow * removed unused files * remove unnecesary files * json2ts * change to debug logs * remove pipeline profiler helpers * Update generated TypeScript types * Update generated TypeScript types * created PipelineExecutionIndex * Fix limit param for pagination * Update generated TypeScript types * addressed comments * linting * fix sample_data * Added serviceType in api response * Add endtime in sample data * Update generated TypeScript types * Addressed comments --------- Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: Harshit Shah <harshit.shah@getcollate.io>
100 lines
No EOL
3.9 KiB
JSON
100 lines
No EOL
3.9 KiB
JSON
{
|
|
"tasks": [{
|
|
"name": "hive_create_table",
|
|
"displayName": "Hive Create Table",
|
|
"description": "Hive Create Table Task",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=hive_create_table",
|
|
"downstreamTasks": ["assert_table_exits"],
|
|
"taskType": "HiveOperator"
|
|
},
|
|
{
|
|
"name": "assert_table_exists",
|
|
"displayName": "Assert Table Exists",
|
|
"description": "Assert if a table exists",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=assert_table_exists",
|
|
"downstreamTasks": [],
|
|
"taskType": "HiveOperator"
|
|
},
|
|
{
|
|
"name": "snowflake_task",
|
|
"displayName": "Snowflake Task",
|
|
"description": "Airflow operator to perform ETL on snowflake tables",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=assert_table_exists",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "SnowflakeOperator"
|
|
},
|
|
{
|
|
"name": "presto_task",
|
|
"displayName": "Presto Task",
|
|
"description": "Airflow operator to perform ETL on presto tables",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=assert_table_exists",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "PrestoOperator"
|
|
},
|
|
{
|
|
"name": "dim_address_task",
|
|
"displayName": "dim_address Task",
|
|
"description": "Airflow operator to perform ETL and generate dim_address table",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=dim_address_task",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "PrestoOperator"
|
|
},
|
|
{
|
|
"name": "dim_user_task",
|
|
"displayName": "dim_user Task",
|
|
"description": "Airflow operator to perform ETL and generate dim_user table",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=dim_user_task",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "PrestoOperator"
|
|
},
|
|
{
|
|
"name": "dim_location_task",
|
|
"displayName": "dim_location Task",
|
|
"description": "Airflow operator to perform ETL and generate dim_location table",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=dim_location_task",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "PrestoOperator"
|
|
},
|
|
{
|
|
"name": "trino_task",
|
|
"displayName": "Trino Task",
|
|
"description": "Airflow operator to perform ETL on trino tables",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=assert_table_exists",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "TrinoOperator"
|
|
},
|
|
{
|
|
"name": "dim_location_etl",
|
|
"displayName": "dim_location etl",
|
|
"description": "dim_location ETL pipeline",
|
|
"sourceUrl": "http://localhost:8080/tree?dag_id=dim_address_etl",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=assert_table_exists",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "TrinoOperator"
|
|
},
|
|
{
|
|
"name": "metrics_aggregation_task",
|
|
"displayName": "Metrics Aggregation Task",
|
|
"description": "Airflow operator to aggregate streaming metrics in real-time",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=metrics_aggregation_task",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "PythonOperator"
|
|
},
|
|
{
|
|
"name": "feature_engineering_task",
|
|
"displayName": "Feature Engineering Task",
|
|
"description": "Airflow operator to generate ML features from warehouse data",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=feature_engineering_task",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "PythonOperator"
|
|
},
|
|
{
|
|
"name": "dim_product_task",
|
|
"displayName": "dim_product Task",
|
|
"description": "Airflow operator to perform ETL and generate dim_product table",
|
|
"sourceUrl": "http://localhost:8080/taskinstance/list/?flt1_dag_id_equals=dim_product_task",
|
|
"downstreamTasks": ["assert_table_exists"],
|
|
"taskType": "PrestoOperator"
|
|
}
|
|
]
|
|
} |