diff --git a/scripts/scaffold/kafka-connect/Dockerfile b/scripts/scaffold/kafka-connect/Dockerfile index 015bf97f41..f6070a0b3a 100644 --- a/scripts/scaffold/kafka-connect/Dockerfile +++ b/scripts/scaffold/kafka-connect/Dockerfile @@ -5,7 +5,10 @@ USER root RUN yum install -y jq findutils unzip RUN confluent-hub install snowflakeinc/snowflake-kafka-connector:2.5.0 --no-prompt +RUN confluent-hub install confluentinc/kafka-connect-jdbc:10.7.4 --no-prompt + COPY tmp/kafka-connect-http/ /usr/share/confluent-hub-components/kafka-connect-http/ +COPY tmp/custom-plugins/tinybird-smt-1.0.0.jar /usr/share/java/tinybird-smt-1.0.0.jar VOLUME /storage diff --git a/services/libs/tinybird/pipes/cdp_dashboard_metrics_total_sink.pipe b/services/libs/tinybird/pipes/cdp_dashboard_metrics_total_sink.pipe new file mode 100644 index 0000000000..26c15991b9 --- /dev/null +++ b/services/libs/tinybird/pipes/cdp_dashboard_metrics_total_sink.pipe @@ -0,0 +1,46 @@ +DESCRIPTION > + Global metrics for activities, organizations, and members used for CDP dashboard. We. are referring to the total witouth filtering by any segment + +NODE activityRelationsMetricsTotal +SQL > + SELECT + count() AS activitiesTotal, + countIf(createdAt >= now() - INTERVAL 30 DAY) AS activitiesLast30Days + FROM activityRelations_deduplicated_ds + +NODE organizationsMetricsTotal +SQL > + SELECT + count() AS organizationsTotal, + countIf(createdAt >= now() - INTERVAL 30 DAY) AS organizationsLast30Days + FROM organizations FINAL + +NODE membersMetricsTotal +SQL > + SELECT count() AS membersTotal, countIf(joinedAt >= now() - INTERVAL 30 DAY) AS membersLast30Days + FROM members FINAL + +NODE mergeResults +SQL > + SELECT + -- activity + (SELECT activitiesTotal FROM activityRelationsMetricsTotal) AS activitiesTotal, + (SELECT activitiesLast30Days FROM activityRelationsMetricsTotal) AS activitiesLast30Days, + -- organizations + (SELECT organizationsTotal FROM organizationsMetricsTotal) AS organizationsTotal, + (SELECT organizationsLast30Days FROM organizationsMetricsTotal) AS organizationsLast30Days, + -- members + (SELECT membersTotal FROM membersMetricsTotal) AS membersTotal, + (SELECT membersLast30Days FROM membersMetricsTotal) AS membersLast30Days + +NODE cdpDashboardFullMetricsTotal +SQL > + SELECT * FROM mergeResults + +TYPE SINK +EXPORT_SERVICE kafka +EXPORT_CONNECTION_NAME lfx-oracle-kafka-streaming-staging +EXPORT_SCHEDULE 30 0 * * * +EXPORT_FORMAT json +EXPORT_STRATEGY @new +EXPORT_KAFKA_TOPIC cdp_dashboard_metrics_total_sink