Jenkinsfile: init
Signed-off-by: Alexis Pentori <alexis@status.im>
This commit is contained in:
parent
fb9a6bf821
commit
3859002d31
|
@ -0,0 +1,58 @@
|
||||||
|
pipeline {
|
||||||
|
agent { label 'linux' }
|
||||||
|
|
||||||
|
options {
|
||||||
|
disableConcurrentBuilds()
|
||||||
|
/* manage how many builds we keep */
|
||||||
|
buildDiscarder(logRotator(
|
||||||
|
numToKeepStr: '20',
|
||||||
|
daysToKeepStr: '30',
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
parameters {
|
||||||
|
string(
|
||||||
|
name: 'DOCKER_CRED',
|
||||||
|
description: 'Name of Docker Registry credential.',
|
||||||
|
defaultValue: params.DOCKER_CRED ?: 'harbor-status-im-robot',
|
||||||
|
)
|
||||||
|
string(
|
||||||
|
name: 'DOCKER_REGISTRY_URL',
|
||||||
|
description: 'URL of the Docker Registry',
|
||||||
|
defaultValue: params.DOCKER_REGISTRY_URL ?: 'https://harbor.status.im'
|
||||||
|
)
|
||||||
|
}
|
||||||
|
stages {
|
||||||
|
stage('Bulding docker images') {
|
||||||
|
steps { script {
|
||||||
|
def directories= findFiles().findAll { item -> item.directory && item.name != '.git' }
|
||||||
|
for (directory in directories) {
|
||||||
|
stage("Building ${directory}"){
|
||||||
|
script {
|
||||||
|
def metadataVal = readYaml file : "./${directory}/metadata.yaml"
|
||||||
|
image_tag = metadataVal['data']['dockerImageTag']
|
||||||
|
image_name = metadataVal['data']['dockerRepository']
|
||||||
|
echo "Building ${directory} - image name : ${image_name}:${image_tag}"
|
||||||
|
}
|
||||||
|
script {
|
||||||
|
image = docker.build(
|
||||||
|
"${image_name}:${image_tag}",
|
||||||
|
"./${directory}"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
script {
|
||||||
|
withDockerRegistry([
|
||||||
|
credentialsId: params.DOCKER_CRED, url: params.DOCKER_REGISTRY_URL
|
||||||
|
]) {
|
||||||
|
image.push()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
post {
|
||||||
|
always { cleanWs()}
|
||||||
|
}
|
||||||
|
}
|
|
@ -10,8 +10,8 @@ data:
|
||||||
connectorSubtype: api
|
connectorSubtype: api
|
||||||
connectorType: source
|
connectorType: source
|
||||||
definitionId: 1c448bfb-8950-478c-9ae0-f03aaaf4e920
|
definitionId: 1c448bfb-8950-478c-9ae0-f03aaaf4e920
|
||||||
dockerImageTag: 0.0.2
|
dockerImageTag: '0.0.3'
|
||||||
dockerRepository: harbor.status.im/status-im/airbyte/source-social-network-fetcher
|
dockerRepository: status-im/airbyte/social-network-fetcher
|
||||||
githubIssueLabel: source-social-network-fetcher
|
githubIssueLabel: source-social-network-fetcher
|
||||||
icon: social-network-fetcher.svg
|
icon: social-network-fetcher.svg
|
||||||
license: MIT
|
license: MIT
|
||||||
|
|
|
@ -68,15 +68,17 @@ class TwitterAccountData(TwitterStream):
|
||||||
logger.info("Response: %s", response.json())
|
logger.info("Response: %s", response.json())
|
||||||
data=response.json()['data']
|
data=response.json()['data']
|
||||||
yield {
|
yield {
|
||||||
"account_id": data['id'],
|
"account_id": data['id'],
|
||||||
"username": data['username'],
|
"username": data['username'],
|
||||||
"account_name": data['name'],
|
"account_name": data['name'],
|
||||||
"tweet_count": data['public_metrics']['tweet_count'],
|
"tweet_count": data['public_metrics']['tweet_count'],
|
||||||
"like_count": data['public_metrics']['like_count'],
|
"like_count": data['public_metrics']['like_count'],
|
||||||
"following_count": data['public_metrics']['following_count'],
|
"following_count": data['public_metrics']['following_count'],
|
||||||
"follower_count": data['public_metrics']['followers_count'],
|
"follower_count": data['public_metrics']['followers_count'],
|
||||||
"listed_count": data['public_metrics']['listed_count'],
|
"listed_count": data['public_metrics']['listed_count'],
|
||||||
}
|
}
|
||||||
|
time.sleep(2)
|
||||||
|
|
||||||
|
|
||||||
class TwitterTweet(HttpSubStream, TwitterAccountData):
|
class TwitterTweet(HttpSubStream, TwitterAccountData):
|
||||||
#TODO: See how to get the account ID
|
#TODO: See how to get the account ID
|
||||||
|
@ -103,20 +105,19 @@ class TwitterTweet(HttpSubStream, TwitterAccountData):
|
||||||
logger.info("Response: %s", response.json())
|
logger.info("Response: %s", response.json())
|
||||||
referenced_tweets=""
|
referenced_tweets=""
|
||||||
for t in data:
|
for t in data:
|
||||||
if "referenced_tweets" in t:
|
if "referenced_tweets" in t:
|
||||||
for rt in t.get('referenced_tweets'):
|
for rt in t.get('referenced_tweets'):
|
||||||
referenced_tweets += f"{rt.get('type')}:{rt.get('id')};"
|
referenced_tweets += f"{rt.get('type')}:{rt.get('id')};"
|
||||||
yield {
|
yield {
|
||||||
"id": t['id'],
|
"id": t['id'],
|
||||||
"created_at": t.get('created_at'),
|
"created_at": t.get('created_at'),
|
||||||
"retweet_count": t.get('public_metrics').get('retweet_count'),
|
"retweet_count": t.get('public_metrics').get('retweet_count'),
|
||||||
"reply_count": t.get('public_metrics').get('reply_count'),
|
"reply_count": t.get('public_metrics').get('reply_count'),
|
||||||
"like_count": t.get('public_metrics').get('like_count'),
|
"like_count": t.get('public_metrics').get('like_count'),
|
||||||
"quote_count": t.get('public_metrics').get('quote_count'),
|
"quote_count": t.get('public_metrics').get('quote_count'),
|
||||||
"referenced_tweets": referenced_tweets
|
"referenced_tweets": referenced_tweets
|
||||||
}
|
}
|
||||||
|
time.sleep(2)
|
||||||
|
|
||||||
|
|
||||||
# Source
|
# Source
|
||||||
class SourceSocialNetworkFetcher(AbstractSource):
|
class SourceSocialNetworkFetcher(AbstractSource):
|
||||||
|
|
|
@ -16,3 +16,6 @@ connectionSpecification:
|
||||||
description: List of accounts needing to be extracted
|
description: List of accounts needing to be extracted
|
||||||
items:
|
items:
|
||||||
type: string
|
type: string
|
||||||
|
start_time:
|
||||||
|
type: string
|
||||||
|
description: Start date of fetching data
|
||||||
|
|
|
@ -11,7 +11,7 @@ data:
|
||||||
connectorType: source
|
connectorType: source
|
||||||
definitionId: 1e55cfe0-f591-4281-9a20-18d89d45f685
|
definitionId: 1e55cfe0-f591-4281-9a20-18d89d45f685
|
||||||
dockerImageTag: 0.7.0
|
dockerImageTag: 0.7.0
|
||||||
dockerRepository: harbor.status.im/status-im/airbyte/wallet-fetcher
|
dockerRepository: status-im/airbyte/wallet-fetcher
|
||||||
githubIssueLabel: source-wallet-fetcher
|
githubIssueLabel: source-wallet-fetcher
|
||||||
icon: icon.svg
|
icon: icon.svg
|
||||||
license: MIT
|
license: MIT
|
||||||
|
|
Loading…
Reference in New Issue