{"version":"0.2.0","built_at":"2026-10-10T20:58:14.188Z","license":"CC BY 4.0 © Mike Reams","canonical":"https://catalog.mikereams.com","capability":{"id":"BC-750.30","name":"Data Engineering & Pipelines","level":2,"parent_id":"BC-750","l1_id":"BC-750","industry":"cross-industry","macro_id":"MC-70","path":["Data & Analytics","Data Engineering & Pipelines"],"description":"Build and run the pipelines that collect, clean, transform and deliver data to where it is used, reliably enough that a missing load is an incident rather than a normal morning.","aliases":["Data Engineering","ETL","Data Integration"],"in_scope":["Data ingestion from internal and external sources","Transformation and modeling in the platform","Pipeline scheduling and monitoring","Data platform operations and cost"],"out_of_scope":["Application-to-application integration (see BC-720)","Infrastructure that hosts the platform (see BC-730)"],"csdm_note":null,"status":"active","successor_id":null,"child_ids":["BC-750.30.10","BC-750.30.20","BC-750.30.30","BC-750.30.40"]},"crosswalk":{"realized_by":[{"product_id":"PR-180","product":"Snowflake","pc_id":"PC-180.10","pc":"Data Warehouse","maturity":"core","via":"BC-750.30.40"},{"product_id":"PR-180","product":"Snowflake","pc_id":"PC-180.20","pc":"Data Engineering","maturity":"strong","via":"BC-750.30.20"},{"product_id":"PR-180","product":"Snowflake","pc_id":"PC-180.30","pc":"Data Sharing and Marketplace","maturity":"core","via":"BC-750.30.10"}],"used_in":[],"patterns":[{"id":"DP-290","name":"Transactional Outbox","category":"Data","via":"BC-750.30.10"},{"id":"DP-300","name":"Change Data Capture","category":"Data","via":"BC-750.30.10"},{"id":"DP-330","name":"Medallion Lakehouse Layering","category":"Data","via":"BC-750.30.20"}],"stacks":[{"id":"TS-030","name":"Open lakehouse analytics platform","layer":"Data platform","via":null}],"books":[{"id":"BK-designing-data-intensive-applications","title":"Designing Data-Intensive Applications","topic":"Distributed systems","via":null},{"id":"BK-data-mesh","title":"Data Mesh","topic":"Data","via":null},{"id":"BK-fundamentals-of-data-engineering","title":"Fundamentals of Data Engineering","topic":"Data","via":null},{"id":"BK-streaming-systems","title":"Streaming Systems","topic":"Data","via":null}]},"ancestors":[{"id":"BC-750","name":"Data & Analytics","level":1}],"children":[{"id":"BC-750.30.10","name":"Data Ingestion","level":3,"child_count":0},{"id":"BC-750.30.20","name":"Data Transformation","level":3,"child_count":0},{"id":"BC-750.30.30","name":"Pipeline Operations","level":3,"child_count":0},{"id":"BC-750.30.40","name":"Data Platform Operations","level":3,"child_count":0}]}