From 3859002d313121fae14796e4108499c8c0a9b8f3 Mon Sep 17 00:00:00 2001 From: Alexis Pentori Date: Tue, 30 Jan 2024 15:15:53 +0100 Subject: [PATCH] Jenkinsfile: init Signed-off-by: Alexis Pentori --- Jenkinsfile | 58 +++++++++++++++++++ source-social-network-fetcher/metadata.yaml | 4 +- .../source_social_network_fetcher/source.py | 47 +++++++-------- .../source_social_network_fetcher/spec.yaml | 3 + wallet-fetcher/metadata.yaml | 2 +- 5 files changed, 88 insertions(+), 26 deletions(-) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..63731ed --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,58 @@ +pipeline { + agent { label 'linux' } + + options { + disableConcurrentBuilds() + /* manage how many builds we keep */ + buildDiscarder(logRotator( + numToKeepStr: '20', + daysToKeepStr: '30', + )) + } + + parameters { + string( + name: 'DOCKER_CRED', + description: 'Name of Docker Registry credential.', + defaultValue: params.DOCKER_CRED ?: 'harbor-status-im-robot', + ) + string( + name: 'DOCKER_REGISTRY_URL', + description: 'URL of the Docker Registry', + defaultValue: params.DOCKER_REGISTRY_URL ?: 'https://harbor.status.im' + ) + } + stages { + stage('Bulding docker images') { + steps { script { + def directories= findFiles().findAll { item -> item.directory && item.name != '.git' } + for (directory in directories) { + stage("Building ${directory}"){ + script { + def metadataVal = readYaml file : "./${directory}/metadata.yaml" + image_tag = metadataVal['data']['dockerImageTag'] + image_name = metadataVal['data']['dockerRepository'] + echo "Building ${directory} - image name : ${image_name}:${image_tag}" + } + script { + image = docker.build( + "${image_name}:${image_tag}", + "./${directory}" + ) + } + script { + withDockerRegistry([ + credentialsId: params.DOCKER_CRED, url: params.DOCKER_REGISTRY_URL + ]) { + image.push() + } + } + } + } + } } + } + } + post { + always { cleanWs()} + } +} diff --git a/source-social-network-fetcher/metadata.yaml b/source-social-network-fetcher/metadata.yaml index 73b95d2..6de1916 100644 --- a/source-social-network-fetcher/metadata.yaml +++ b/source-social-network-fetcher/metadata.yaml @@ -10,8 +10,8 @@ data: connectorSubtype: api connectorType: source definitionId: 1c448bfb-8950-478c-9ae0-f03aaaf4e920 - dockerImageTag: 0.0.2 - dockerRepository: harbor.status.im/status-im/airbyte/source-social-network-fetcher + dockerImageTag: '0.0.3' + dockerRepository: status-im/airbyte/social-network-fetcher githubIssueLabel: source-social-network-fetcher icon: social-network-fetcher.svg license: MIT diff --git a/source-social-network-fetcher/source_social_network_fetcher/source.py b/source-social-network-fetcher/source_social_network_fetcher/source.py index 4f3aa57..41cbc21 100644 --- a/source-social-network-fetcher/source_social_network_fetcher/source.py +++ b/source-social-network-fetcher/source_social_network_fetcher/source.py @@ -68,15 +68,17 @@ class TwitterAccountData(TwitterStream): logger.info("Response: %s", response.json()) data=response.json()['data'] yield { - "account_id": data['id'], - "username": data['username'], - "account_name": data['name'], - "tweet_count": data['public_metrics']['tweet_count'], - "like_count": data['public_metrics']['like_count'], - "following_count": data['public_metrics']['following_count'], - "follower_count": data['public_metrics']['followers_count'], - "listed_count": data['public_metrics']['listed_count'], - } + "account_id": data['id'], + "username": data['username'], + "account_name": data['name'], + "tweet_count": data['public_metrics']['tweet_count'], + "like_count": data['public_metrics']['like_count'], + "following_count": data['public_metrics']['following_count'], + "follower_count": data['public_metrics']['followers_count'], + "listed_count": data['public_metrics']['listed_count'], + } + time.sleep(2) + class TwitterTweet(HttpSubStream, TwitterAccountData): #TODO: See how to get the account ID @@ -103,20 +105,19 @@ class TwitterTweet(HttpSubStream, TwitterAccountData): logger.info("Response: %s", response.json()) referenced_tweets="" for t in data: - if "referenced_tweets" in t: - for rt in t.get('referenced_tweets'): - referenced_tweets += f"{rt.get('type')}:{rt.get('id')};" - yield { - "id": t['id'], - "created_at": t.get('created_at'), - "retweet_count": t.get('public_metrics').get('retweet_count'), - "reply_count": t.get('public_metrics').get('reply_count'), - "like_count": t.get('public_metrics').get('like_count'), - "quote_count": t.get('public_metrics').get('quote_count'), - "referenced_tweets": referenced_tweets - } - - + if "referenced_tweets" in t: + for rt in t.get('referenced_tweets'): + referenced_tweets += f"{rt.get('type')}:{rt.get('id')};" + yield { + "id": t['id'], + "created_at": t.get('created_at'), + "retweet_count": t.get('public_metrics').get('retweet_count'), + "reply_count": t.get('public_metrics').get('reply_count'), + "like_count": t.get('public_metrics').get('like_count'), + "quote_count": t.get('public_metrics').get('quote_count'), + "referenced_tweets": referenced_tweets + } + time.sleep(2) # Source class SourceSocialNetworkFetcher(AbstractSource): diff --git a/source-social-network-fetcher/source_social_network_fetcher/spec.yaml b/source-social-network-fetcher/source_social_network_fetcher/spec.yaml index db56885..e6673fb 100644 --- a/source-social-network-fetcher/source_social_network_fetcher/spec.yaml +++ b/source-social-network-fetcher/source_social_network_fetcher/spec.yaml @@ -16,3 +16,6 @@ connectionSpecification: description: List of accounts needing to be extracted items: type: string + start_time: + type: string + description: Start date of fetching data diff --git a/wallet-fetcher/metadata.yaml b/wallet-fetcher/metadata.yaml index ee49592..d2a9232 100644 --- a/wallet-fetcher/metadata.yaml +++ b/wallet-fetcher/metadata.yaml @@ -11,7 +11,7 @@ data: connectorType: source definitionId: 1e55cfe0-f591-4281-9a20-18d89d45f685 dockerImageTag: 0.7.0 - dockerRepository: harbor.status.im/status-im/airbyte/wallet-fetcher + dockerRepository: status-im/airbyte/wallet-fetcher githubIssueLabel: source-wallet-fetcher icon: icon.svg license: MIT