Jenkinsfile: init

Signed-off-by: Alexis Pentori <alexis@status.im>

Jenkinsfile

@@ -0,0 +1,58 @@
+pipeline {
+  agent { label 'linux' }
+  
+  options {
+    disableConcurrentBuilds()
+    /* manage how many builds we keep */
+    buildDiscarder(logRotator(
+      numToKeepStr: '20',
+      daysToKeepStr: '30',
+    ))
+  }
+
+  parameters {
+    string(
+      name: 'DOCKER_CRED',
+      description: 'Name of Docker Registry credential.',
+      defaultValue: params.DOCKER_CRED ?: 'harbor-status-im-robot',
+    )
+    string(
+      name: 'DOCKER_REGISTRY_URL',
+      description: 'URL of the Docker Registry',
+      defaultValue: params.DOCKER_REGISTRY_URL ?: 'https://harbor.status.im'
+    )
+  }
+  stages {
+    stage('Bulding docker images') {
+      steps { script {
+        def directories=  findFiles().findAll { item -> item.directory && item.name != '.git' }
+        for (directory in directories) {
+          stage("Building ${directory}"){
+            script {
+             def metadataVal = readYaml file : "./${directory}/metadata.yaml"
+             image_tag = metadataVal['data']['dockerImageTag']
+             image_name = metadataVal['data']['dockerRepository']
+             echo "Building ${directory} - image name : ${image_name}:${image_tag}"
+            }
+            script {
+              image = docker.build(
+                "${image_name}:${image_tag}",
+                "./${directory}"
+              )
+            }
+            script {
+               withDockerRegistry([
+                 credentialsId: params.DOCKER_CRED, url: params.DOCKER_REGISTRY_URL
+               ]) {
+                 image.push()
+              }
+            }
+          }
+        }
+      } } 
+    }
+  }
+  post {
+      always { cleanWs()}
+  }
+}

source-social-network-fetcher/metadata.yaml

@@ -10,8 +10,8 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 1c448bfb-8950-478c-9ae0-f03aaaf4e920
-  dockerImageTag: 0.0.2
+  dockerImageTag: '0.0.3'
-  dockerRepository: harbor.status.im/status-im/airbyte/source-social-network-fetcher
+  dockerRepository: status-im/airbyte/social-network-fetcher
   githubIssueLabel: source-social-network-fetcher
   icon: social-network-fetcher.svg
   license: MIT

source-social-network-fetcher/source_social_network_fetcher/source.py

@@ -68,15 +68,17 @@ class TwitterAccountData(TwitterStream):
     logger.info("Response: %s", response.json())
     data=response.json()['data']
     yield {
-        "account_id": data['id'],
+      "account_id": data['id'],
-        "username": data['username'],
+      "username": data['username'],
-        "account_name": data['name'],
+      "account_name": data['name'],
-        "tweet_count": data['public_metrics']['tweet_count'],
+      "tweet_count": data['public_metrics']['tweet_count'],
-        "like_count": data['public_metrics']['like_count'],
+      "like_count": data['public_metrics']['like_count'],
-        "following_count": data['public_metrics']['following_count'],
+      "following_count": data['public_metrics']['following_count'],
-        "follower_count": data['public_metrics']['followers_count'],
+      "follower_count": data['public_metrics']['followers_count'],
-        "listed_count": data['public_metrics']['listed_count'],
+      "listed_count": data['public_metrics']['listed_count'],
-        }
+    }
+    time.sleep(2)
+
 
 class TwitterTweet(HttpSubStream, TwitterAccountData):
     #TODO: See how to get the account ID
@@ -103,20 +105,19 @@ class TwitterTweet(HttpSubStream, TwitterAccountData):
     logger.info("Response: %s", response.json())
     referenced_tweets=""
     for t in data:
-        if "referenced_tweets" in t:
+      if "referenced_tweets" in t:
-            for rt in t.get('referenced_tweets'):
+        for rt in t.get('referenced_tweets'):
-                referenced_tweets += f"{rt.get('type')}:{rt.get('id')};"
+          referenced_tweets += f"{rt.get('type')}:{rt.get('id')};"
-        yield {
+      yield {
-                "id": t['id'],
+        "id": t['id'],
-                "created_at": t.get('created_at'),
+        "created_at": t.get('created_at'),
-                "retweet_count": t.get('public_metrics').get('retweet_count'),
+        "retweet_count": t.get('public_metrics').get('retweet_count'),
-                "reply_count": t.get('public_metrics').get('reply_count'),
+        "reply_count": t.get('public_metrics').get('reply_count'),
-                "like_count": t.get('public_metrics').get('like_count'),
+        "like_count": t.get('public_metrics').get('like_count'),
-                "quote_count": t.get('public_metrics').get('quote_count'),
+        "quote_count": t.get('public_metrics').get('quote_count'),
-                "referenced_tweets": referenced_tweets
+        "referenced_tweets": referenced_tweets
-            }
+      }
-
+    time.sleep(2)
-
 
 # Source
 class SourceSocialNetworkFetcher(AbstractSource):

source-social-network-fetcher/source_social_network_fetcher/spec.yaml

@@ -16,3 +16,6 @@ connectionSpecification:
       description: List of accounts needing to be extracted
       items:
         type: string
+    start_time:
+      type: string
+      description: Start date of fetching data

wallet-fetcher/metadata.yaml

@@ -11,7 +11,7 @@ data:
   connectorType: source
   definitionId: 1e55cfe0-f591-4281-9a20-18d89d45f685
   dockerImageTag: 0.7.0
-  dockerRepository: harbor.status.im/status-im/airbyte/wallet-fetcher
+  dockerRepository: status-im/airbyte/wallet-fetcher
   githubIssueLabel: source-wallet-fetcher
   icon: icon.svg
   license: MIT