{"id":4698,"date":"2021-10-26T20:52:33","date_gmt":"2021-10-26T15:22:33","guid":{"rendered":"https:\/\/cashfree.com\/blog\/?p=4698"},"modified":"2023-03-07T18:56:45","modified_gmt":"2023-03-07T13:26:45","slug":"building-data-pipeline","status":"publish","type":"post","link":"https:\/\/blogrevamp.cashfree.com\/building-data-pipeline\/","title":{"rendered":"HULK: Building Near Real-Time Data Pipeline For Democratization of Data"},"content":{"rendered":"<div id=\"ez-toc-container\" class=\"ez-toc-v2_0_81 counter-hierarchy ez-toc-counter ez-toc-custom ez-toc-container-direction\">\n<p class=\"ez-toc-title\" style=\"cursor:inherit\">Table of Contents<\/p>\n<label for=\"ez-toc-cssicon-toggle-item-6a8f1db5939a0\" class=\"ez-toc-cssicon-toggle-label\"><span class=\"\"><span class=\"eztoc-hide\" style=\"display:none;\">Toggle<\/span><span class=\"ez-toc-icon-toggle-span\"><svg style=\"fill: #364250;color:#364250\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" class=\"list-377408\" width=\"20px\" height=\"20px\" viewBox=\"0 0 24 24\" fill=\"none\"><path d=\"M6 6H4v2h2V6zm14 0H8v2h12V6zM4 11h2v2H4v-2zm16 0H8v2h12v-2zM4 16h2v2H4v-2zm16 0H8v2h12v-2z\" fill=\"currentColor\"><\/path><\/svg><svg style=\"fill: #364250;color:#364250\" class=\"arrow-unsorted-368013\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"10px\" height=\"10px\" viewBox=\"0 0 24 24\" version=\"1.2\" baseProfile=\"tiny\"><path d=\"M18.2 9.3l-6.2-6.3-6.2 6.3c-.2.2-.3.4-.3.7s.1.5.3.7c.2.2.4.3.7.3h11c.3 0 .5-.1.7-.3.2-.2.3-.5.3-.7s-.1-.5-.3-.7zM5.8 14.7l6.2 6.3 6.2-6.3c.2-.2.3-.5.3-.7s-.1-.5-.3-.7c-.2-.2-.4-.3-.7-.3h-11c-.3 0-.5.1-.7.3-.2.2-.3.5-.3.7s.1.5.3.7z\"\/><\/svg><\/span><\/span><\/label><input type=\"checkbox\"  id=\"ez-toc-cssicon-toggle-item-6a8f1db5939a0\"  aria-label=\"Toggle\" \/><nav><ul class='ez-toc-list ez-toc-list-level-1 ' ><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-1\" href=\"https:\/\/blogrevamp.cashfree.com\/building-data-pipeline\/#Why_Did_We_Need_A_Data_Warehouse\" >Why Did We Need A Data Warehouse?<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-2\" href=\"https:\/\/blogrevamp.cashfree.com\/building-data-pipeline\/#Context_To_Building_the_Data_Warehouse_How_We_Started_Off\" >Context To Building the Data Warehouse: How We Started Off<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-3\" href=\"https:\/\/blogrevamp.cashfree.com\/building-data-pipeline\/#Looking_For_A_Solution_Our_Experiments_with_Platforms\" >Looking For A Solution: Our Experiments with Platforms<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-4\" href=\"https:\/\/blogrevamp.cashfree.com\/building-data-pipeline\/#Introducing_HULK_How_It_Met_Our_Goals\" >Introducing HULK: How It Met Our Goals<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-5\" href=\"https:\/\/blogrevamp.cashfree.com\/building-data-pipeline\/#Internal_Architecture_of_HULK_Our_Data_Warehouse\" >Internal Architecture of HULK, Our Data Warehouse<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-6\" href=\"https:\/\/blogrevamp.cashfree.com\/building-data-pipeline\/#First_Principles\" >First Principles<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-7\" href=\"https:\/\/blogrevamp.cashfree.com\/building-data-pipeline\/#Focusing_On_Batches\" >Focusing On Batches<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-8\" href=\"https:\/\/blogrevamp.cashfree.com\/building-data-pipeline\/#Database_Migrations\" >Database Migrations<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-9\" href=\"https:\/\/blogrevamp.cashfree.com\/building-data-pipeline\/#Alerting_and_Dashboards\" >Alerting and Dashboards<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-10\" href=\"https:\/\/blogrevamp.cashfree.com\/building-data-pipeline\/#Our_Deployment_Setup\" >Our Deployment Setup<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-11\" href=\"https:\/\/blogrevamp.cashfree.com\/building-data-pipeline\/#How_HULK_Helped_Us_Build_a_Near_Real-Time_Data_Pipeline\" >How HULK Helped Us Build a Near Real-Time Data Pipeline<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-12\" href=\"https:\/\/blogrevamp.cashfree.com\/building-data-pipeline\/#The_Road_Ahead\" >The Road Ahead<\/a><\/li><\/ul><\/nav><\/div>\n\t\t<div data-elementor-type=\"wp-post\" data-elementor-id=\"4698\" class=\"elementor elementor-4698\">\n\t\t\t\t\t\t<section class=\"elementor-section elementor-top-section elementor-element elementor-element-20b01b65 elementor-section-boxed elementor-section-height-default elementor-section-height-default\" data-id=\"20b01b65\" data-element_type=\"section\" data-e-type=\"section\">\n\t\t\t\t\t\t<div class=\"elementor-container elementor-column-gap-default\">\n\t\t\t\t\t<div class=\"elementor-column elementor-col-100 elementor-top-column elementor-element elementor-element-12c7d485\" data-id=\"12c7d485\" data-element_type=\"column\" data-e-type=\"column\">\n\t\t\t<div class=\"elementor-widget-wrap elementor-element-populated\">\n\t\t\t\t\t\t<div class=\"elementor-element elementor-element-719b9784 elementor-widget elementor-widget-text-editor\" data-id=\"719b9784\" data-element_type=\"widget\" data-e-type=\"widget\" data-widget_type=\"text-editor.default\">\n\t\t\t\t<div class=\"elementor-widget-container\">\n\t\t\t\t\t\t\t\t\t\n<p class=\"wp-block-paragraph\">Cashfree Payments has seen exponential growth in the last couple of years.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">In fact, the number of merchants on our platform has increased by 150% since 2020! What\u2019s more, we make an average of 20 million API requests every day.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">However, this growth brings a plethora of challenges.&nbsp;&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Now, one of those biggest challenges was <strong>making our data available to everyone in the <a href=\"https:\/\/www.cashfree.com\/?utm_source=blog&amp;utm_medium=organic&amp;utm_campaign=building_data_pipeline_blog\">Cashfree Payments<\/a> team<\/strong>.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><em>And that too <\/em>&#8211; with low latency and ensuring 100% consistency.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Needless to say, we had quite a big task ahead of us. But thankfully, we have a knack for these things!<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">This is our story of building a scalable near real-time data pipeline for the democratization of data.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>And it starts with our motivations for creating it.<\/strong><\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><em>P.S. We codenamed our new data warehouse pipeline HULK. Read on to know why!&nbsp;<\/em><\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Why_Did_We_Need_A_Data_Warehouse\"><\/span>Why Did We Need A Data Warehouse?<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">Well, a quite a few reasons:<\/p>\n\n\n\n<ul class=\"wp-block-list\"><li>We required near real-time data analytics<br><\/li><li>Need to put data in the centre of decision making<br><\/li><li>Remove dependency on MySQL replicas (and use them for their intended purpose: read-heavy queries for services)<br><\/li><li>Reducing costs; for every RDS instance, we had to provision additional replicas to support our ever-growing analytics requirements.<br><\/li><li>Improving the availability of data for all the teams<\/li><\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">As of then, we were working exclusively with MySQL read replicas as our primary data source for analytics and data processing.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">But now that we knew what we wanted out of our upcoming data warehouse, we had a clearer idea of how to start.&nbsp;&nbsp;<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Context_To_Building_the_Data_Warehouse_How_We_Started_Off\"><\/span>Context To Building the Data Warehouse: How We Started Off<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">Cashfree started as a monolith built with PHP, with all data residing in Primary RDS managed MySQL instances.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">As we scaled our teams and offerings, the components were split into several services and thus multiple database instances.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">However, we faced a lot of issues while doing that.&nbsp;<\/p>\n\n\n\n<ul class=\"wp-block-list\"><li>No capability to join tables; information across multiple databases.<br><\/li><li>Scaling big queries on MySQL replicas is hard, given production databases might not have additional indexes, etc required for analytics queries.<br><\/li><li>Some analytics query could trigger full table scans. This could lead to Replica scanning billions of rows resulting in high replication lag.&nbsp;<\/li><\/ul>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/lh5.googleusercontent.com\/EoGx_9QNqYPYA8IgEjJbJrVjfT7efK6VCWa4yJ05vC3CWdde-iStpIdTI1DWPBteJJc5-phqRQtfTksL93tHw5bV1jYEpHvv_zNME2Ccia_JoUNLc4DFWJA-4Cox9n-UX7FC_dzw=s1600\" alt=\"HULK: Building Near Real-Time Data Pipeline For Democratization of Data\"\/><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\">We knew that we needed an out-of-the-box solution to meet our objectives.<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Looking_For_A_Solution_Our_Experiments_with_Platforms\"><\/span>Looking For A Solution: Our Experiments with Platforms<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">We began experimenting with CDC platforms and Kafka connectors in late 2020 to move from RDS to RedShift for warehouse and analytics across Cashfree.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">During peak times, one of our primary databases generates 1M+ updates per minute. This leads to a similar number of database events being sent. There are other systems that perform reconciliations, bank statements etc. They generate a surge of even higher updates.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Most of the systems we experimented with were not able to flexibly handle spikes (including data migration solutions and different Kafka Sink providers.)<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">This resulted in high latency for data availability in RedShift, <strong>as high as 10-12 hours<\/strong>.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Another major problem with existing solutions was this:<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Ability to replay or restore batches.&nbsp;<\/strong><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">RedShift does not apply any constraints on primary keys (since it has none) on the incoming data.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">This can cause issues if batches fail, or are replayed for some reason. Since existing tools mostly apply messages from CDC, an insert event delivered twice would result in a double entry for the same record.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">This can be <strong>very risky<\/strong>, specifically for us at Cashfree Payments since we use the same source for financial reporting and risk analysis.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">So, here is how we decided to go about this.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">We split ourselves into two groups:&nbsp;<\/p>\n\n\n\n<ol class=\"wp-block-list\"><li>One focuses on finding the right available choice that can help us gain low latency and predictable sync.<\/li><li>And the other group focuses on building a simple service to help us achieve our goals.<\/li><\/ol>\n\n\n\n<p class=\"wp-block-paragraph\">Our goals being:<\/p>\n\n\n\n<ul class=\"wp-block-list\"><li>Ability to handle millions of updates on multiple databases<\/li><li>Near real-time; we set aggressive goals of under 5 minutes for our busiest database.&nbsp;<\/li><li>Keeping costs lower than existing solutions&nbsp;<\/li><li>Handling schema migrations and database version changes&nbsp;<\/li><li>Providing replayable and idempotent batches&nbsp;<\/li><\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">After spending a few weeks iterating on various options, we were ready with the first version of the service.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>We codenamed it HULK and let\u2019s just say, the name was quite fitting!<\/strong><\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Introducing_HULK_How_It_Met_Our_Goals\"><\/span>Introducing HULK: How It Met Our Goals<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">In our current setup, we have multiple MySQL databases, connected to the CDC platform which sends the updates on Kafka cluster.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">From the cluster, HULK instances read the updates, batches and send them to Redshift using the COPY command.\u00a0<\/p>\n\n\n\n<figure class=\"wp-block-image size-large\"><img fetchpriority=\"high\" decoding=\"async\" data-attachment-id=\"4840\" data-permalink=\"https:\/\/blogrevamp.cashfree.com\/building-data-pipeline\/hulk-04\/\" data-orig-file=\"https:\/\/i0.wp.com\/blogrevamp.cashfree.com\/wp-content\/uploads\/2021\/10\/Hulk-04.png?fit=1200%2C628&amp;ssl=1\" data-orig-size=\"1200,628\" data-comments-opened=\"1\" data-image-meta=\"{&quot;aperture&quot;:&quot;0&quot;,&quot;credit&quot;:&quot;&quot;,&quot;camera&quot;:&quot;&quot;,&quot;caption&quot;:&quot;&quot;,&quot;created_timestamp&quot;:&quot;0&quot;,&quot;copyright&quot;:&quot;&quot;,&quot;focal_length&quot;:&quot;0&quot;,&quot;iso&quot;:&quot;0&quot;,&quot;shutter_speed&quot;:&quot;0&quot;,&quot;title&quot;:&quot;&quot;,&quot;orientation&quot;:&quot;0&quot;}\" data-image-title=\"Hulk-04\" data-image-description=\"\" data-image-caption=\"\" data-large-file=\"https:\/\/i0.wp.com\/blogrevamp.cashfree.com\/wp-content\/uploads\/2021\/10\/Hulk-04.png?fit=770%2C403&amp;ssl=1\" data-recalc-dims=\"1\" height=\"403\" width=\"770\" src=\"https:\/\/i0.wp.com\/cashfree.com\/blog\/wp-content\/uploads\/2021\/10\/Hulk-04-1024x536.png?resize=770%2C403&#038;ssl=1\" alt=\"data warehouse high level overview\" class=\"wp-image-4840\" srcset=\"https:\/\/i0.wp.com\/blogrevamp.cashfree.com\/wp-content\/uploads\/2021\/10\/Hulk-04.png?resize=1024%2C536&amp;ssl=1 1024w, https:\/\/i0.wp.com\/blogrevamp.cashfree.com\/wp-content\/uploads\/2021\/10\/Hulk-04.png?resize=300%2C157&amp;ssl=1 300w, https:\/\/i0.wp.com\/blogrevamp.cashfree.com\/wp-content\/uploads\/2021\/10\/Hulk-04.png?resize=768%2C402&amp;ssl=1 768w, https:\/\/i0.wp.com\/blogrevamp.cashfree.com\/wp-content\/uploads\/2021\/10\/Hulk-04.png?w=1200&amp;ssl=1 1200w\" sizes=\"(max-width: 770px) 100vw, 770px\" \/><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\">Speaking of HULK, let\u2019s have a look at how it works.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>And most importantly, why it worked for us.<\/strong><\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Internal_Architecture_of_HULK_Our_Data_Warehouse\"><\/span>Internal Architecture of HULK, Our Data Warehouse<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">Our aim was to build something that adds value, saves us the cost and time of monitoring\/scaling.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Naturally, we had to ensure that we were moving in the right direction- every step of the way.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">So, we decided to stick to the following principles.&nbsp;&nbsp;<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"First_Principles\"><\/span>First Principles<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<ul class=\"wp-block-list\"><li><strong>Automation:<\/strong> The time unit tests save and allow fast iteration is always underrated. This helped us move fast with confidence.<br><\/li><li><strong>Metrics:<\/strong> Observability is critical, especially for systems such as HULK. We started with a dashboard first and then made sure we added similar metrics while building it. It gave us visibility of systems at runtime.<br><\/li><li><strong>Fail-Fast:<\/strong> We didn\u2019t want to make any assumptions. After all, it&#8217;s better to fail instead of being wrong.<\/li><\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">We experimented with SQL libraries and updates to only realise Redshift does not work well with single statement updates.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Hence, we focused on batches. Specifically on <strong>how to run batches efficiently on redshift<\/strong>.&nbsp;<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Focusing_On_Batches\"><\/span>Focusing On Batches<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">One of the best ways of moving data into redshift is by using the <a href=\"https:\/\/docs.aws.amazon.com\/redshift\/latest\/dg\/r_COPY.html\">COPY<\/a> command. This helps to quickly move millions of records to redshift with a single command.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">However, as with everything in engineering&#8211; <strong>it comes with a trade-off.&nbsp;<\/strong><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Every copy and subsequent actions has overhead. Hence to make things more efficient, we needed to define the right batch sizes, wait times etc to optimize operations.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Now that we had resolved the issue of moving data at an optimized speed, our next big challenge was to <strong>handle source database migrations.<\/strong>&nbsp;<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Database_Migrations\"><\/span>Database Migrations<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">We had our fair share of challenges implementing fair migrations from MySQL.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">For starters:<\/p>\n\n\n\n<ul class=\"wp-block-list\"><li>For any CDC event that is an Alter, we fetch source table configurations and compare with RedShift schema and make a decision.<br><\/li><li>We had to be cautious of redshift types and defaults, few examples:&nbsp;<ul><li>Default precision\/scale of double is (11,0) which can cause data loss.<\/li><li>TEXT is simply translated to VARCHAR(256) instead of MAX.<br><\/li><\/ul><\/li><li>The focus for the most part is to do in-place migrations since re-creating tables can be very expensive and time-consuming.<br><\/li><li>We took extra caution before applying any updates since we want batches to be replayable.<br><\/li><\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">It\u2019s important that we had the right visibility and observability of the system; especially its performance during runtime. This adheres to our tight bound SLAs.&nbsp;&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">After all, gauging metrics was one of our primary motivations while building the system.<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Alerting_and_Dashboards\"><\/span>Alerting and Dashboards<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">Now, we have dashboards with real-time status for&nbsp;<\/p>\n\n\n\n<ul class=\"wp-block-list\"><li>99%ile lag in seconds from MySQL, each database has different accepted SLA based on size and alerts on top of it.&nbsp;<\/li><li>Kafka lag on the CDC topics<\/li><li>Number of messages produced per minute<\/li><li>Redshift batch times, broken down into each stage of COPY command<\/li><li>Total number of batches being processed per minute.&nbsp;<\/li><li>Every DDL applied and translated, while the system is very stable but we still do cross-check translated SQLs.&nbsp;&nbsp;<\/li><\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">With alerting in place, we had to ensure that our deployment strategy had an elastic scale so that it could adjust to runtime demands.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Speaking of which\u2026<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Our_Deployment_Setup\"><\/span>Our Deployment Setup<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">Our deployment setup itself could take up the form of a whole other blog.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">But here are some pointers for a quick overview:<\/p>\n\n\n\n<ul class=\"wp-block-list\"><li>We set up a separate Kubernetes cluster for the data platform which hosts CDC, Kafka and HULK instances.&nbsp;<\/li><li>We usually have one instance per database which helps us avoid cascading failures across multiple databases. This setup also helps us manage instance size in proportion to the updates we expect from the database.&nbsp;<\/li><li>We used Prometheus for scraping the metrics and Grafana for visualization.&nbsp;<\/li><li>We integrated with Pagerduty and Slack for alerts.<\/li><\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">We have already discussed the HULK architecture and its functionalities.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">It\u2019s only fitting to discuss the results!<\/p>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"How_HULK_Helped_Us_Build_a_Near_Real-Time_Data_Pipeline\"><\/span>How HULK Helped Us Build a Near Real-Time Data Pipeline<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">We have seen tremendous benefits on the overall ecosystem since we have moved to HULK.&nbsp;&nbsp;<\/p>\n\n\n\n<ul class=\"wp-block-list\"><li>Our costs are a fraction of what we had using data migration services and sink connectors (including VPC etc).<br><\/li><li>We have more visibility\/control on data movements and fine alerting mechanisms on more contextual events\/batches.<br><\/li><li>We have billions of rows moved into Redshift via H. This has enabled our finance, risk and product teams to have a more meaningful and connected data ecosystem built on top of it.<br><\/li><li>Average and Peak CPU utilization of Redshift has dropped considerably from consistently being over 80-85% to around 60-65%. This gives us more room to strategise our platform plans without worrying a lot about redshift being a bottleneck<\/li><\/ul>\n\n\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"The_Road_Ahead\"><\/span>The Road Ahead<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">Yes, this has been a big win for us.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">However, it&#8217;s just the first step towards our goal of building an awesome data engineering team.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Of course, we have a long road ahead of us.&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Here are our focus points for the near future:<\/p>\n\n\n\n<ul class=\"wp-block-list\"><li>Auto VACUUM and exploring if we can run vacuum on high throughput tables in the night via HULK.<br><\/li><li>Alerts\/visibility on whether we have the right sort and dist keys for our tables based on usage patterns.<br><\/li><li>Purging tables, for tables where historical data does not have a lot of context.<br><\/li><li>Optimizing file sizes for S3, compressions and parallel copy.&nbsp;<\/li><\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">We are excited for all our future experiments, <em>to say the least.<\/em><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Does this sound like something that would interest you?&nbsp;<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Well then, we have some exciting opportunities lined up for data engineers just like you!&nbsp;<\/p>\n\n\n\n<figure class=\"wp-block-image\"><a href=\"https:\/\/www.cashfree.com\/careers?utm_source=blog&amp;utm_medium=organic&amp;utm_campaign=building_data_pipeline_blog\"><img decoding=\"async\" src=\"https:\/\/lh4.googleusercontent.com\/rSnB02BfGW7ZPQ8wcZv9sLqOdzaFJDGvcxdu40fvYeFToyDxZMD1Fzu46_P6W-f0tyf_5SjZxO20lrGXqfeMcjGKWl8uF5ug1IDjkIBpLD768vlUES_79UXwJ1Qtvmc1FHMHdARU=s1600\" alt=\"\"\/><\/a><\/figure>\n\t\t\t\t\t\t\t\t<\/div>\n\t\t\t\t<\/div>\n\t\t\t\t\t<\/div>\n\t\t<\/div>\n\t\t\t\t\t<\/div>\n\t\t<\/section>\n\t\t\t\t<\/div>\n\t\t","protected":false},"excerpt":{"rendered":"<p>Cashfree Payments has seen exponential growth in the last couple of years.&nbsp; In fact, the number of merchants on our platform has increased by 150% since 2020! What\u2019s more, we make an average of 20 million API requests every day. However, this growth brings a plethora of challenges.&nbsp;&nbsp; Now, one of those biggest challenges was<\/p>\n","protected":false},"author":26,"featured_media":7134,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_exactmetrics_skip_tracking":false,"_exactmetrics_sitenote_active":false,"_exactmetrics_sitenote_note":"","_exactmetrics_sitenote_category":0,"_themeisle_gutenberg_block_has_review":false,"_jetpack_newsletter_access":"","_jetpack_dont_email_post_to_subs":false,"_jetpack_newsletter_tier_id":0,"_jetpack_memberships_contains_paywalled_content":false,"_wpcom_ai_launchpad_first_post":false,"_jetpack_feature_clip_id":0,"_jetpack_memberships_contains_paid_content":false,"footnotes":"","jetpack_post_was_ever_published":false},"categories":[1,1497],"tags":[1522,1737],"class_list":["post-4698","post","type-post","status-publish","format-standard","has-post-thumbnail","category-all","category-engineering","tag-cashfree-payments","tag-tech-stories"],"acf":[],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v28.3 - https:\/\/yoast.com\/product\/yoast-seo-wordpress\/ -->\n<title>HULK: Building Near Real-Time Data Pipeline For Democratization of Data<\/title>\n<meta name=\"description\" content=\"We built a scalable near real-time data pipeline for the democratization of data and codenamed it HULK. Read on to know why!\u00a0\" \/>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.cashfree.com\/blog\/building-data-pipeline\/\" \/>\n<meta property=\"og:locale\" content=\"en_US\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"HULK: Building Near Real-Time Data Pipeline For Democratization of Data\" \/>\n<meta property=\"og:description\" content=\"We built a scalable near real-time data pipeline for the democratization of data and codenamed it HULK. Read on to know why!\u00a0\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.cashfree.com\/blog\/building-data-pipeline\/\" \/>\n<meta property=\"og:site_name\" content=\"Cashfree Payments Blog\" \/>\n<meta property=\"article:published_time\" content=\"2021-10-26T15:22:33+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2023-03-07T13:26:45+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/i0.wp.com\/blogrevamp.cashfree.com\/wp-content\/uploads\/2021\/10\/Hulk_Thumbnail-2.png?fit=1400%2C720&ssl=1\" \/>\n\t<meta property=\"og:image:width\" content=\"1400\" \/>\n\t<meta property=\"og:image:height\" content=\"720\" \/>\n\t<meta property=\"og:image:type\" content=\"image\/png\" \/>\n<meta name=\"author\" content=\"Supal Dubey\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"Written by\" \/>\n\t<meta name=\"twitter:data1\" content=\"Supal Dubey\" \/>\n\t<meta name=\"twitter:label2\" content=\"Est. reading time\" \/>\n\t<meta name=\"twitter:data2\" content=\"8 minutes\" \/>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"HULK: Building Near Real-Time Data Pipeline For Democratization of Data","description":"We built a scalable near real-time data pipeline for the democratization of data and codenamed it HULK. Read on to know why!\u00a0","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.cashfree.com\/blog\/building-data-pipeline\/","og_locale":"en_US","og_type":"article","og_title":"HULK: Building Near Real-Time Data Pipeline For Democratization of Data","og_description":"We built a scalable near real-time data pipeline for the democratization of data and codenamed it HULK. Read on to know why!\u00a0","og_url":"https:\/\/www.cashfree.com\/blog\/building-data-pipeline\/","og_site_name":"Cashfree Payments Blog","article_published_time":"2021-10-26T15:22:33+00:00","article_modified_time":"2023-03-07T13:26:45+00:00","og_image":[{"width":1400,"height":720,"url":"https:\/\/i0.wp.com\/blogrevamp.cashfree.com\/wp-content\/uploads\/2021\/10\/Hulk_Thumbnail-2.png?fit=1400%2C720&ssl=1","type":"image\/png"}],"author":"Supal Dubey","twitter_card":"summary_large_image","twitter_misc":{"Written by":"Supal Dubey","Est. reading time":"8 minutes"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"Article","@id":"https:\/\/www.cashfree.com\/blog\/building-data-pipeline\/#article","isPartOf":{"@id":"https:\/\/blogrevamp.cashfree.com\/building-data-pipeline\/"},"author":{"name":"Supal Dubey","@id":"https:\/\/blogrevamp.cashfree.com\/#\/schema\/person\/56e8785573831cd3ed450a32a6413f7f"},"headline":"HULK: Building Near Real-Time Data Pipeline For Democratization of Data","datePublished":"2021-10-26T15:22:33+00:00","dateModified":"2023-03-07T13:26:45+00:00","mainEntityOfPage":{"@id":"https:\/\/blogrevamp.cashfree.com\/building-data-pipeline\/"},"wordCount":1733,"commentCount":0,"image":{"@id":"https:\/\/www.cashfree.com\/blog\/building-data-pipeline\/#primaryimage"},"thumbnailUrl":"https:\/\/i0.wp.com\/blogrevamp.cashfree.com\/wp-content\/uploads\/2021\/10\/Hulk_Thumbnail-2.png?fit=1400%2C720&ssl=1","keywords":["cashfree payments","tech stories"],"articleSection":["All","Engineering"],"inLanguage":"en-US","potentialAction":[{"@type":"CommentAction","name":"Comment","target":["https:\/\/www.cashfree.com\/blog\/building-data-pipeline\/#respond"]}]},{"@type":"WebPage","@id":"https:\/\/blogrevamp.cashfree.com\/building-data-pipeline\/","url":"https:\/\/www.cashfree.com\/blog\/building-data-pipeline\/","name":"HULK: Building Near Real-Time Data Pipeline For Democratization of Data","isPartOf":{"@id":"https:\/\/blogrevamp.cashfree.com\/#website"},"primaryImageOfPage":{"@id":"https:\/\/www.cashfree.com\/blog\/building-data-pipeline\/#primaryimage"},"image":{"@id":"https:\/\/www.cashfree.com\/blog\/building-data-pipeline\/#primaryimage"},"thumbnailUrl":"https:\/\/i0.wp.com\/blogrevamp.cashfree.com\/wp-content\/uploads\/2021\/10\/Hulk_Thumbnail-2.png?fit=1400%2C720&ssl=1","datePublished":"2021-10-26T15:22:33+00:00","dateModified":"2023-03-07T13:26:45+00:00","author":{"@id":"https:\/\/blogrevamp.cashfree.com\/#\/schema\/person\/56e8785573831cd3ed450a32a6413f7f"},"description":"We built a scalable near real-time data pipeline for the democratization of data and codenamed it HULK. Read on to know why!\u00a0","breadcrumb":{"@id":"https:\/\/www.cashfree.com\/blog\/building-data-pipeline\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.cashfree.com\/blog\/building-data-pipeline\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/www.cashfree.com\/blog\/building-data-pipeline\/#primaryimage","url":"https:\/\/i0.wp.com\/blogrevamp.cashfree.com\/wp-content\/uploads\/2021\/10\/Hulk_Thumbnail-2.png?fit=1400%2C720&ssl=1","contentUrl":"https:\/\/i0.wp.com\/blogrevamp.cashfree.com\/wp-content\/uploads\/2021\/10\/Hulk_Thumbnail-2.png?fit=1400%2C720&ssl=1","width":1400,"height":720},{"@type":"BreadcrumbList","@id":"https:\/\/www.cashfree.com\/blog\/building-data-pipeline\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https:\/\/blogrevamp.cashfree.com\/"},{"@type":"ListItem","position":2,"name":"All","item":"https:\/\/blogrevamp.cashfree.com\/category\/all\/"},{"@type":"ListItem","position":3,"name":"HULK: Building Near Real-Time Data Pipeline For Democratization of Data"}]},{"@type":"WebSite","@id":"https:\/\/blogrevamp.cashfree.com\/#website","url":"https:\/\/blogrevamp.cashfree.com\/","name":"Cashfree Payments Blog","description":"Cashfree Payments- Payment Gateway for India","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/blogrevamp.cashfree.com\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Person","@id":"https:\/\/blogrevamp.cashfree.com\/#\/schema\/person\/56e8785573831cd3ed450a32a6413f7f","name":"Supal Dubey","url":"https:\/\/blogrevamp.cashfree.com\/author\/supalcf\/"}]}},"jetpack_sharing_enabled":true,"jetpack_shortlink":"https:\/\/wp.me\/p9MjXo-1dM","jetpack_likes_enabled":false,"jetpack_featured_media_url":"https:\/\/i0.wp.com\/blogrevamp.cashfree.com\/wp-content\/uploads\/2021\/10\/Hulk_Thumbnail-2.png?fit=1400%2C720&ssl=1","_links":{"self":[{"href":"https:\/\/blogrevamp.cashfree.com\/wp-json\/wp\/v2\/posts\/4698","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/blogrevamp.cashfree.com\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/blogrevamp.cashfree.com\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/blogrevamp.cashfree.com\/wp-json\/wp\/v2\/users\/26"}],"replies":[{"embeddable":true,"href":"https:\/\/blogrevamp.cashfree.com\/wp-json\/wp\/v2\/comments?post=4698"}],"version-history":[{"count":7,"href":"https:\/\/blogrevamp.cashfree.com\/wp-json\/wp\/v2\/posts\/4698\/revisions"}],"predecessor-version":[{"id":16117,"href":"https:\/\/blogrevamp.cashfree.com\/wp-json\/wp\/v2\/posts\/4698\/revisions\/16117"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/blogrevamp.cashfree.com\/wp-json\/wp\/v2\/media\/7134"}],"wp:attachment":[{"href":"https:\/\/blogrevamp.cashfree.com\/wp-json\/wp\/v2\/media?parent=4698"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/blogrevamp.cashfree.com\/wp-json\/wp\/v2\/categories?post=4698"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/blogrevamp.cashfree.com\/wp-json\/wp\/v2\/tags?post=4698"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}