Jenkinsfile: init

Signed-off-by: Alexis Pentori <alexis@status.im>
This commit is contained in:
Alexis Pentori 2024-01-30 15:15:53 +01:00
parent fb9a6bf821
commit 3859002d31
5 changed files with 88 additions and 26 deletions

58
Jenkinsfile vendored Normal file
View File

@ -0,0 +1,58 @@
pipeline {
agent { label 'linux' }
options {
disableConcurrentBuilds()
/* manage how many builds we keep */
buildDiscarder(logRotator(
numToKeepStr: '20',
daysToKeepStr: '30',
))
}
parameters {
string(
name: 'DOCKER_CRED',
description: 'Name of Docker Registry credential.',
defaultValue: params.DOCKER_CRED ?: 'harbor-status-im-robot',
)
string(
name: 'DOCKER_REGISTRY_URL',
description: 'URL of the Docker Registry',
defaultValue: params.DOCKER_REGISTRY_URL ?: 'https://harbor.status.im'
)
}
stages {
stage('Bulding docker images') {
steps { script {
def directories= findFiles().findAll { item -> item.directory && item.name != '.git' }
for (directory in directories) {
stage("Building ${directory}"){
script {
def metadataVal = readYaml file : "./${directory}/metadata.yaml"
image_tag = metadataVal['data']['dockerImageTag']
image_name = metadataVal['data']['dockerRepository']
echo "Building ${directory} - image name : ${image_name}:${image_tag}"
}
script {
image = docker.build(
"${image_name}:${image_tag}",
"./${directory}"
)
}
script {
withDockerRegistry([
credentialsId: params.DOCKER_CRED, url: params.DOCKER_REGISTRY_URL
]) {
image.push()
}
}
}
}
} }
}
}
post {
always { cleanWs()}
}
}

View File

@ -10,8 +10,8 @@ data:
connectorSubtype: api
connectorType: source
definitionId: 1c448bfb-8950-478c-9ae0-f03aaaf4e920
dockerImageTag: 0.0.2
dockerRepository: harbor.status.im/status-im/airbyte/source-social-network-fetcher
dockerImageTag: '0.0.3'
dockerRepository: status-im/airbyte/social-network-fetcher
githubIssueLabel: source-social-network-fetcher
icon: social-network-fetcher.svg
license: MIT

View File

@ -68,15 +68,17 @@ class TwitterAccountData(TwitterStream):
logger.info("Response: %s", response.json())
data=response.json()['data']
yield {
"account_id": data['id'],
"username": data['username'],
"account_name": data['name'],
"tweet_count": data['public_metrics']['tweet_count'],
"like_count": data['public_metrics']['like_count'],
"following_count": data['public_metrics']['following_count'],
"follower_count": data['public_metrics']['followers_count'],
"listed_count": data['public_metrics']['listed_count'],
}
"account_id": data['id'],
"username": data['username'],
"account_name": data['name'],
"tweet_count": data['public_metrics']['tweet_count'],
"like_count": data['public_metrics']['like_count'],
"following_count": data['public_metrics']['following_count'],
"follower_count": data['public_metrics']['followers_count'],
"listed_count": data['public_metrics']['listed_count'],
}
time.sleep(2)
class TwitterTweet(HttpSubStream, TwitterAccountData):
#TODO: See how to get the account ID
@ -103,20 +105,19 @@ class TwitterTweet(HttpSubStream, TwitterAccountData):
logger.info("Response: %s", response.json())
referenced_tweets=""
for t in data:
if "referenced_tweets" in t:
for rt in t.get('referenced_tweets'):
referenced_tweets += f"{rt.get('type')}:{rt.get('id')};"
yield {
"id": t['id'],
"created_at": t.get('created_at'),
"retweet_count": t.get('public_metrics').get('retweet_count'),
"reply_count": t.get('public_metrics').get('reply_count'),
"like_count": t.get('public_metrics').get('like_count'),
"quote_count": t.get('public_metrics').get('quote_count'),
"referenced_tweets": referenced_tweets
}
if "referenced_tweets" in t:
for rt in t.get('referenced_tweets'):
referenced_tweets += f"{rt.get('type')}:{rt.get('id')};"
yield {
"id": t['id'],
"created_at": t.get('created_at'),
"retweet_count": t.get('public_metrics').get('retweet_count'),
"reply_count": t.get('public_metrics').get('reply_count'),
"like_count": t.get('public_metrics').get('like_count'),
"quote_count": t.get('public_metrics').get('quote_count'),
"referenced_tweets": referenced_tweets
}
time.sleep(2)
# Source
class SourceSocialNetworkFetcher(AbstractSource):

View File

@ -16,3 +16,6 @@ connectionSpecification:
description: List of accounts needing to be extracted
items:
type: string
start_time:
type: string
description: Start date of fetching data

View File

@ -11,7 +11,7 @@ data:
connectorType: source
definitionId: 1e55cfe0-f591-4281-9a20-18d89d45f685
dockerImageTag: 0.7.0
dockerRepository: harbor.status.im/status-im/airbyte/wallet-fetcher
dockerRepository: status-im/airbyte/wallet-fetcher
githubIssueLabel: source-wallet-fetcher
icon: icon.svg
license: MIT