init

Signed-off-by: Alexis Pentori <alexis@status.im>

.gitignore

@@ -0,0 +1 @@
+postgres/data

Makefile

@@ -0,0 +1,13 @@
+.PHONY: up down dbt-build dbt-compile
+
+up:
+	docker compose up -d
+
+down:
+	dockerc compose down
+
+dbt-build:
+	docker compose run dbt build
+
+dbt-compile:
+	docker compose run dbt compile

README.md

@@ -0,0 +1,46 @@
+# BI Recruitment Task
+
+## Description
+
+This repo provide a set of preconfigured tools that can be used for the recruitment task for IFT:
+
+* PostgreSQL database
+* DBT
+* Grafana
+
+## Tasks
+
+Based on the data available at the remote database, design and implement a dashboard to help a Project Manager :
+  * Have an overview of the project progress
+  * Identify improvment
+  * Evaluate the project adoption
+
+The data are base on the project [Waku](https://github.com/waku-org/).
+
+The database configuration:
+* host: `recruitment.free.technology`
+* port: `5432`
+* user: will be provided
+* password: will be provided
+* database name: `recuitment_task`
+* schemas: `raw_github`,`raw_finance`
+
+It is recommended to submit the tasks result as a git repository containing all the files to reproduce the result.
+> If you are using this repo, we recommand that you export the grafana dashboard and add it with the dbt models to the git history.
+
+This repo contain a preconfigured partial stack to let you focus on the data manipulation. You are free to use any free technology you like, on the condition we can easily reproduce the result.
+
+A partical solution is fine too, as long as you document what is missing or needs improvment.
+
+## Requirements
+
+* Have docker installed
+
+## Usage
+
+* Deploy the container with `make run`
+* Shutdown the containers with `make down`
+* Build the dbt models with `make dbt-buidlt`
+* Compile the dbt models with `make dbt-compile`
+
+The data from the database and grafana are persisted with docker volumes.

dbt_project/.gitignore

@@ -0,0 +1,4 @@
+
+target/
+dbt_packages/
+logs/

dbt_project/.user.yml

@@ -0,0 +1 @@
+id: 596b42c5-0af9-4191-8bd2-9b791288b55b

dbt_project/README.md

@@ -0,0 +1,15 @@
+Welcome to your new dbt project!
+
+### Using the starter project
+
+Try running the following commands:
+- dbt run
+- dbt test
+
+
+### Resources:
+- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction)
+- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
+- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support
+- Find [dbt events](https://events.getdbt.com) near you
+- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices

dbt_project/dbt_project.yml

@@ -0,0 +1,36 @@
+
+# Name your project! Project names should contain only lowercase characters
+# and underscores. A good package name should reflect your organization's
+# name or the intended use of these models
+name: 'dbt_project'
+version: '1.0.0'
+
+# This setting configures which "profile" dbt uses for this project.
+profile: 'dbt_project'
+
+# These configurations specify where dbt should look for different types of files.
+# The `model-paths` config, for example, states that models in this project can be
+# found in the "models/" directory. You probably won't need to change these!
+model-paths: ["models"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+clean-targets:         # directories to be removed by `dbt clean`
+  - "target"
+  - "dbt_packages"
+
+
+# Configuring models
+# Full documentation: https://docs.getdbt.com/docs/configuring-models
+
+# In this example config, we tell dbt to build all models in the example/
+# directory as views. These settings can be overridden in the individual model
+# files using the `{{ config(...) }}` macro.
+models:
+  dbt_project:
+    # Config indicated by + and applies to all files under models/example/
+    example:
+      +materialized: view

dbt_project/models/example/my_first_dbt_model.sql

@@ -0,0 +1,27 @@
+
+/*
+    Welcome to your first dbt model!
+    Did you know that you can also configure models directly within SQL files?
+    This will override configurations stated in dbt_project.yml
+
+    Try changing "table" to "view" below
+*/
+
+{{ config(materialized='table') }}
+
+with source_data as (
+
+    select 1 as id
+    union all
+    select null as id
+
+)
+
+select *
+from source_data
+
+/*
+    Uncomment the line below to remove records with null `id` values
+*/
+
+-- where id is not null

dbt_project/models/example/my_second_dbt_model.sql

@@ -0,0 +1,6 @@
+
+-- Use the `ref` function to select from other models
+
+select *
+from {{ ref('my_first_dbt_model') }}
+where id = 1

dbt_project/models/example/schema.yml

@@ -0,0 +1,21 @@
+
+version: 2
+
+models:
+  - name: my_first_dbt_model
+    description: "A starter dbt model"
+    columns:
+      - name: id
+        description: "The primary key for this table"
+        data_tests:
+          - unique
+          - not_null
+
+  - name: my_second_dbt_model
+    description: "A starter dbt model"
+    columns:
+      - name: id
+        description: "The primary key for this table"
+        data_tests:
+          - unique
+          - not_null

dbt_project/profiles.yml

@@ -0,0 +1,12 @@
+dbt_project:
+  outputs:
+    dev:
+      dbname: postgres
+      host: database
+      pass: Password
+      port: 5432
+      schema: public
+      threads: 1
+      type: postgres
+      user: admin
+  target: dev

docker-compose.yml

@@ -0,0 +1,52 @@
+---
+services:
+  database:
+    image: 'postgres:15'
+    container_name: 'database'
+    environment:
+      POSTGRES_DB: postgres
+      POSTGRES_PASSWORD: Password
+      POSTGRES_USER: admin
+      PGDATA: /var/lib/postgresql/data/pgdata
+    volumes:
+      - "./postgres/data:/var/lib/postgresql/data"
+      - "./postgres/init/:/docker-entrypoint-initdb.d"
+    ports:
+      - '5432:5432'
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U admin -d postgres"]
+      interval: 10s
+      retries: 5
+      start_period: 30s
+      timeout: 10s
+
+  dbt:
+    container_name: dbt
+    image: ghcr.io/dbt-labs/dbt-postgres:1.7.2 #dbtlabs/dbt:latest  #dbt-dummy
+    build: dbt_project/
+    volumes:
+      - ./dbt_project:/usr/app/dbt
+    ports:
+      - "8080:8080"
+    environment:
+      DBT_PROFILES_DIR: /usr/app/dbt
+      DBT_TARGET: dev
+    depends_on:
+      - database
+    command: ["run"]
+
+  grafana:
+    image: 'grafana/grafana-enterprise'
+    container_name: 'grafana'
+    volumes:
+      - "grafana-data:/var/lib/grafana"
+      - "./grafana/grafana.ini:/etc/grafana/grafana.ini"
+      - "./grafana/grafana-datasource-provisioning.yml:/etc/grafana/provisioning/datasources/postgres.yml"
+    ports:
+      - '3000:3000'
+    depends_on:
+      database:
+        condition: service_healthy
+
+volumes:
+  grafana-data:

grafana/grafana-datasource-provisioning.yml

@@ -0,0 +1,17 @@
+apiVersion: 1
+
+datasources:
+  - name: Postgres
+    type: postgres
+    url: database:5432
+    user: admin
+    secureJsonData:
+      password: 'Password'
+    jsonData:
+      database: postgres
+      sslmode: 'disable' # disable/require/verify-ca/verify-full
+      maxOpenConns: 100 # Grafana v5.4+
+      maxIdleConns: 100 # Grafana v5.4+
+      maxIdleConnsAuto: true # Grafana v9.5.1+
+      connMaxLifetime: 14400 # Grafana v5.4+
+      timescaledb: false

grafana/grafana.ini

@@ -0,0 +1,3 @@
+[security]
+admin_user = admin
+admin_password = Password!

postgres/init/test.sql

@@ -0,0 +1,27 @@
+CREATE TABLE demo (
+	id INT,
+	number INT,
+	name VARCHAR,
+	description VARCHAR,
+	insert_date DATE
+);
+INSERT INTO demo (id, number, name, description, insert_date)
+VALUES (18, 36453, 'Landon', 'physics electronic issn damage tri', '1986-07-24T17:34:56.923Z');
+INSERT INTO demo (id, number, name, description, insert_date)
+VALUES (60, 81473, 'Cody', 'sparc dutch barbie players poor', '2001-09-09T13:45:45.532Z');
+INSERT INTO demo (id, number, name, description, insert_date)
+VALUES (80, 32744, 'Shaffer', 'draft directories balloon institution companion', '1993-06-07T08:50:21.187Z');
+INSERT INTO demo (id, number, name, description, insert_date)
+VALUES (95, 98852, 'Mcclintock', 'collection heard place rpm struct', '2013-05-21T03:17:41.849Z');
+INSERT INTO demo (id, number, name, description, insert_date)
+VALUES (43, 58217, 'Evans', 'ev scsi addressed gone weather', '2011-11-12T20:32:17.498Z');
+INSERT INTO demo (id, number, name, description, insert_date)
+VALUES (71, 15643, 'Nash', 'assess columnists gold accessibility colleague', '1981-01-05T06:28:16.580Z');
+INSERT INTO demo (id, number, name, description, insert_date)
+VALUES (57, 52371, 'Mulkey', 'delta vii afraid debut predicted', '2014-07-05T22:52:27.639Z');
+INSERT INTO demo (id, number, name, description, insert_date)
+VALUES (19, 35288, 'Becnel', 'immunology milk latex pat mobiles', '1973-12-07T18:41:15.948Z');
+INSERT INTO demo (id, number, name, description, insert_date)
+VALUES (62, 51569, 'Bingham', 'holes analysts edited stick undergraduate', '1996-12-27T23:40:17.710Z');
+INSERT INTO demo (id, number, name, description, insert_date)
+VALUES (72, 85567, 'Armstead', 'upper suddenly such thoroughly hotel', '1970-10-21T16:32:11.706Z');