status-im
/
airflow-dags
mirror of https://github.com/status-im/airflow-dags.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

comm_extraction: add status, ift, and operator accounts 

Signed-off-by: Alexis Pentori <alexis@status.im>
This commit is contained in:
Alexis Pentori 2024-10-24 11:33:27 +02:00
parent 11c93275a1
commit b273f250d2
No known key found for this signature in database
GPG Key ID: 65250D2801E47A10
1 changed files with 9 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
comm_extraction.py
View File

@ -40,10 +40,13 @@ airbyte_connections=[
'twitter-nomos',
'twitter-codex',
'twitter-logos',
'twitter-status',
'twitter-nimbus',
'twitter-waku',
'twitter-vac',
'twitter-keycard',
'twitter-ift',
'twitter-operator',
]
@ -98,7 +101,7 @@ def comm_extraction():
twitter_logos = fetch_twitter_info(connections_id['twitter-logos'], 'logos')
# twitter_status = fetch_twitter_info(connections_id['twitter-status'], 'status')
twitter_status = fetch_twitter_info(connections_id['twitter-status'], 'status')
twitter_nimbus = fetch_twitter_info(connections_id['twitter-nimbus'], 'nimbus')
@ -108,6 +111,10 @@ def comm_extraction():
twitter_keycard = fetch_twitter_info(connections_id['twitter-keycard'], 'keycard')
twitter_ift = fetch_twitter_info(connections_id['twitter-ift'], 'ift')
twitter_operator = fetch_twitter_info(connections_id['twitter-operator'], 'operator')
dbt_run_socials = BashOperator(
task_id='dbt_run_models_social',
bash_command='dbt run --profiles-dir /dbt --project-dir /dbt/dbt-models/ --select social'
@ -118,7 +125,7 @@ def comm_extraction():
)
# Twitter connections have to be sequentially run to avoid API Rate Limits
connections_id >> [discord_fetcher, simplecast_fetch] >> twitter_nomos_tech >> twitter_codex >> twitter_logos >> twitter_waku >> twitter_nimbus >> twitter_vac >> twitter_keycard >> dbt_run_socials >> dbt_run_twitter
connections_id >> [discord_fetcher, simplecast_fetch] >> twitter_nomos_tech >> twitter_codex >> twitter_logos >> twitter_waku >> twitter_nimbus >> twitter_vac >> twitter_keycard >> twitter_operator >> twitter_ift >> dbt_run_socials >> dbt_run_twitter
comm_extraction()