discorse-fecther: init connector
Signed-off-by: Alexis Pentori <alexis@status.im>
This commit is contained in:
Alexis Pentori
parent
636dacea08
commit
bf464bbed0
|
|
@ -0,0 +1,8 @@
|
||||||
|
FROM airbyte/python-connector-base:1.1.0
|
||||||
|
|
||||||
|
COPY . ./airbyte/integration_code
|
||||||
|
RUN pip install ./airbyte/integration_code
|
||||||
|
|
||||||
|
# The entrypoint and default env vars are already set in the base image
|
||||||
|
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
|
||||||
|
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
|
||||||
|
|
@ -0,0 +1,62 @@
|
||||||
|
# Discourse Fetcher Source
|
||||||
|
|
||||||
|
This is the repository for fetching data from Discourse forum, written in Python.
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
This connector fecth user and post data from a discourse forum instance.
|
||||||
|
|
||||||
|
### Configuration
|
||||||
|
|
||||||
|
The connector takes the following input:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
- api-key
|
||||||
|
- api-username
|
||||||
|
- url
|
||||||
|
```
|
||||||
|
|
||||||
|
### Output
|
||||||
|
|
||||||
|
The connector will return the following:
|
||||||
|
- `posts`: List of post on the discourse instance.
|
||||||
|
- `users`: List of user on the discourse instance.
|
||||||
|
|
||||||
|
## Local development
|
||||||
|
|
||||||
|
### Prerequisites
|
||||||
|
|
||||||
|
#### Activate Virtual Environment and install dependencies
|
||||||
|
From this connector directory, create a virtual environment:
|
||||||
|
```
|
||||||
|
python -m venv .venv
|
||||||
|
```
|
||||||
|
```
|
||||||
|
source .venv/bin/activate
|
||||||
|
pip install -r requirements.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
### Locally running the connector
|
||||||
|
```
|
||||||
|
python main.py spec
|
||||||
|
python main.py check --config sample_files/config-example.json
|
||||||
|
python main.py discover --config sample_files/config-example.json
|
||||||
|
python main.py read --config sample_files/config-example.json --catalog sample_files/configured_catalog.json
|
||||||
|
```
|
||||||
|
|
||||||
|
### Locally running the connector docker image
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker build -t airbyte/twitter-fetcher:dev .
|
||||||
|
# Running the spec command against your patched connector
|
||||||
|
docker run airbyte/twitter-fetcher:dev spec
|
||||||
|
````
|
||||||
|
|
||||||
|
#### Run
|
||||||
|
Then run any of the connector commands as follows:
|
||||||
|
```
|
||||||
|
docker run --rm airbyte/twitter-fetcher:dev spec
|
||||||
|
docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/twitter-fetcher:dev check --config /sample_files/config-example.json
|
||||||
|
docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/twitter-fetcher:dev discover --config /sample_files/config-example.json
|
||||||
|
docker run --rm -v $(pwd)/sample_files:/sample_files -v $(pwd)/sample_files:/sample_files airbyte/twitter-fetcher:dev read --config /sample_files/config-example.json --catalog /sample_files/configured_catalog.json
|
||||||
|
```
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<svg version="1.1" xmlns="http://www.w3.org/2000/svg" width="128" height="128">
|
||||||
|
<path d="M0 0 C42.24 0 84.48 0 128 0 C128 42.24 128 84.48 128 128 C85.76 128 43.52 128 0 128 C0 85.76 0 43.52 0 0 Z " fill="#221E1F" transform="translate(0,0)"/>
|
||||||
|
<path d="M0 0 C0.63421875 0.25523438 1.2684375 0.51046875 1.921875 0.7734375 C8.34108613 3.76358285 10.66291542 9.54703772 13.11328125 15.875 C16.20354096 26.76578313 13.77564469 37.65528085 8.71484375 47.5 C2.9440993 57.01610226 -6.29206836 63.07301709 -17 65.75 C-22.66295983 66.22191332 -28.02462001 66.16044019 -33.640625 65.27734375 C-39.55405277 64.66478002 -44.05675657 66.27405519 -49.5 68.4375 C-51.25961426 69.10498373 -53.01999398 69.77045417 -54.78125 70.43359375 C-55.54501953 70.73322021 -56.30878906 71.03284668 -57.09570312 71.34155273 C-59 72 -59 72 -61 72 C-60.48972622 68.12487737 -59.13333532 65.12779461 -57.32055664 61.68554688 C-53.5216374 54.93458227 -53.5216374 54.93458227 -53.5625 47.5 C-53.80355469 46.81679687 -54.04460938 46.13359375 -54.29296875 45.4296875 C-57.01229128 36.08483328 -55.50149242 25.49689988 -51 17 C-45.32394943 7.853618 -37.38639409 1.01003291 -27.09375 -2.34375 C-17.9594016 -4.28133906 -8.50731057 -3.97699478 0 0 Z " fill="#F1E88F" transform="translate(85,32)"/>
|
||||||
|
<path d="M0 0 C17.49 0 34.98 0 53 0 C53 42.24 53 84.48 53 128 C34.85 128 16.7 128 -2 128 C-2 127.67 -2 127.34 -2 127 C-1.29617187 126.86464844 -0.59234375 126.72929688 0.1328125 126.58984375 C18.33383873 122.6217349 31.82800874 113.09190516 42.31640625 97.828125 C51.09679557 83.07870072 53.60728183 65.67937284 50.3125 48.83984375 C45.77255688 31.75830777 35.34288889 17.28676573 20.08203125 8.27734375 C13.59960432 4.86421599 7.14032683 2.60235681 0 1 C0 0.67 0 0.34 0 0 Z " fill="#FFFFFF" transform="translate(75,0)"/>
|
||||||
|
<path d="M0 0 C3.97940152 1.49884947 5.43463485 3.74712589 7.625 7.3125 C8.51058594 8.72982422 8.51058594 8.72982422 9.4140625 10.17578125 C14.69113866 19.57313609 14.5453179 31.25810142 12.16015625 41.6640625 C7.99522403 52.51812828 0.40994583 60.0234585 -10 65 C-15.17950877 67.1730123 -19.84791009 67.33422598 -25.375 67.3125 C-26.18710938 67.32861328 -26.99921875 67.34472656 -27.8359375 67.36132812 C-32.82212237 67.36532347 -36.52227737 66.60044627 -41.10205078 64.67456055 C-44.15149203 63.5907419 -47.04255106 63.67935134 -50.25 63.75 C-51.43078125 63.76546875 -52.6115625 63.7809375 -53.828125 63.796875 C-56.4183852 63.96275373 -58.52067823 64.33207927 -61 65 C-63.33300933 65.03888349 -65.66709801 65.0448635 -68 65 C-62.88055313 59.88055313 -51.99940111 57.21686538 -44.90234375 56.83203125 C-42.56475596 57.10987449 -40.30516955 57.52896174 -38 58 C-26.47464792 58.79686245 -17.22782654 55.19180632 -7.75 48.8125 C0.39021147 40.48728373 4.05176019 30.49753498 4.4375 19 C4.33249639 13.34215824 3.30328252 8.98327306 0.875 3.890625 C0 2 0 2 0 0 Z " fill="#059336" transform="translate(94,39)"/>
|
||||||
|
<path d="M0 0 C2.764409 2.10621638 5.43095207 4.27854636 8.05078125 6.5625 C7.39078125 6.8925 6.73078125 7.2225 6.05078125 7.5625 C4.28125 6.8203125 4.28125 6.8203125 2.23828125 5.6875 C-5.79288677 1.2447262 -16.22453321 1.36865854 -25.05859375 3.2109375 C-36.75707255 7.07738388 -44.43825498 14.87769076 -50.046875 25.67578125 C-53.27688567 33.65315643 -53.38219251 44.00074391 -50.51171875 52.0625 C-48.6027418 58.7118895 -53.88337056 65.88694201 -56.94921875 71.5625 C-57.27921875 71.5625 -57.60921875 71.5625 -57.94921875 71.5625 C-57.80407466 70.08326554 -57.65805453 68.60411704 -57.51171875 67.125 C-57.42792969 66.20847656 -57.34414063 65.29195313 -57.2578125 64.34765625 C-56.94921875 61.5625 -56.94921875 61.5625 -56.484375 59.24609375 C-55.37136505 53.32064077 -56.84286429 50.04225873 -59.734375 44.8359375 C-61.23858537 42.02098433 -61.74613688 39.74411603 -61.94921875 36.5625 C-61.99949219 35.92183594 -62.04976562 35.28117187 -62.1015625 34.62109375 C-62.60155258 23.44131546 -59.28115769 13.96809474 -51.94921875 5.5625 C-37.41332453 -9.27675201 -17.29393958 -12.28699082 0 0 Z " fill="#D13239" transform="translate(82.94921875,27.4375)"/>
|
||||||
|
<path d="M0 0 C17.49 0 34.98 0 53 0 C53 0.33 53 0.66 53 1 C52.36835937 1.13535156 51.73671875 1.27070313 51.0859375 1.41015625 C34.20184985 5.47304189 19.75573896 14.5027621 10 29 C5.6700232 36.33445049 2.8621844 43.70184495 1 52 C0.67 52 0.34 52 0 52 C0 34.84 0 17.68 0 0 Z " fill="#FFFFFF" transform="translate(0,0)"/>
|
||||||
|
<path d="M0 0 C5.14359343 4.42920545 8.27533698 9.62857427 9.39990234 16.41235352 C9.57062615 18.89335595 9.60684329 21.3261426 9.5625 23.8125 C9.54606445 25.12408081 9.54606445 25.12408081 9.52929688 26.4621582 C9.1731192 37.15837173 5.68240319 45.2817738 -1.75390625 53.02734375 C-10.05762042 60.32017098 -18.46733686 63.35537603 -29.375 63.3125 C-30.18710938 63.32861328 -30.99921875 63.34472656 -31.8359375 63.36132812 C-35.92311783 63.36460311 -39.19511726 63.09564564 -42.9765625 61.4921875 C-47.64124934 59.52746388 -52.00408837 59.66285288 -57 59.8125 C-58.30517578 59.83280273 -58.30517578 59.83280273 -59.63671875 59.85351562 C-61.75808627 59.88848322 -63.87912974 59.9422747 -66 60 C-60.51800176 56.34533451 -54.48426441 55.53683826 -48 56 C-46.80375 56.33 -45.6075 56.66 -44.375 57 C-34.32218964 59.67816183 -24.35373075 58.33465571 -15 54 C-5.97579285 48.71939312 -0.79953834 41.5670427 3 32 C3.33 31.34 3.66 30.68 4 30 C4.26517649 26.96493454 4.23159901 23.92036338 4.25 20.875 C4.270625 20.04484375 4.29125 19.2146875 4.3125 18.359375 C4.3490615 12.52781627 3.31799356 8.35011644 1 3 C0.505 1.515 0.505 1.515 0 0 Z " fill="#01AFEC" transform="translate(98,43)"/>
|
||||||
|
<path d="M0 0 C1.96484375 1.65234375 1.96484375 1.65234375 4.77734375 4.46484375 C4.11734375 4.79484375 3.45734375 5.12484375 2.77734375 5.46484375 C1.0078125 4.72265625 1.0078125 4.72265625 -1.03515625 3.58984375 C-9.06632427 -0.85293005 -19.49797071 -0.72899771 -28.33203125 1.11328125 C-40.03051005 4.97972763 -47.71169248 12.78003451 -53.3203125 23.578125 C-56.55032317 31.55550018 -56.65563001 41.90308766 -53.78515625 49.96484375 C-51.8761793 56.61423325 -57.15680806 63.78928576 -60.22265625 69.46484375 C-60.55265625 69.46484375 -60.88265625 69.46484375 -61.22265625 69.46484375 C-61.60165145 64.28832396 -59.74354452 61.34389116 -57.21875 56.96484375 C-55.53397549 52.73639009 -57.14128035 49.53776562 -58.78515625 45.52734375 C-61.84992272 36.48401807 -61.36720332 26.07146367 -57.22265625 17.46484375 C-52.4750252 8.05949891 -44.99836954 0.55414572 -35.22265625 -3.53515625 C-24.86801921 -6.3428903 -9.0316305 -6.23801544 0 0 Z " fill="#E63A2E" transform="translate(86.22265625,29.53515625)"/>
|
||||||
|
<path d="M0 0 C0 0.66 0 1.32 0 2 C-1.65 2.66 -3.3 3.32 -5 4 C-4.01 4.66 -3.02 5.32 -2 6 C-7.30440877 8.65220438 -15.7730822 7.55486301 -21.375 6 C-28.9387437 3.91345001 -34.78523432 5.56120597 -42 8 C-42 8.33 -42 8.66 -42 9 C-44.31 9 -46.62 9 -49 9 C-43.88055313 3.88055313 -32.99940111 1.21686538 -25.90234375 0.83203125 C-23.56475596 1.10987449 -21.30516955 1.52896174 -19 2 C-12.28158021 2.46451132 -6.44850512 0 0 0 Z " fill="#0D963C" transform="translate(75,95)"/>
|
||||||
|
<path d="M0 0 C-0.99 0.33 -1.98 0.66 -3 1 C-3 1.66 -3 2.32 -3 3 C-3.60585938 2.96519531 -4.21171875 2.93039063 -4.8359375 2.89453125 C-11.42183303 2.68175616 -16.81117204 3.7878232 -23 6 C-23 6.33 -23 6.66 -23 7 C-25.31 7 -27.62 7 -30 7 C-23.11424111 0.11424111 -9.04431669 -2.04226506 0 0 Z " fill="#209548" transform="translate(56,97)"/>
|
||||||
|
<path d="M0 0 C4.59191315 3.95414743 7.14607445 8.21036657 9 14 C9 15.32 9 16.64 9 18 C8.34 18.33 7.68 18.66 7 19 C6.34227572 22.02934491 6.34227572 22.02934491 6 25 C5.67 25 5.34 25 5 25 C4.96261719 24.0821875 4.92523438 23.164375 4.88671875 22.21875 C4.50991128 15.18501055 3.90890946 9.42384172 1 3 C0.67 2.01 0.34 1.02 0 0 Z " fill="#04B0DD" transform="translate(98,43)"/>
|
||||||
|
<path d="M0 0 C0 0.99 0 1.98 0 3 C-6.93 3 -13.86 3 -21 3 C-14.89734787 -1.06843475 -7.00766335 -2.33588778 0 0 Z " fill="#02B2DF" transform="translate(53,100)"/>
|
||||||
|
<path d="M0 0 C4.95 0 9.9 0 15 0 C15 0.66 15 1.32 15 2 C10.05 2.33 5.1 2.66 0 3 C0 2.01 0 1.02 0 0 Z " fill="#008F37" transform="translate(53,98)"/>
|
||||||
|
<path d="M0 0 C2.31 0 4.62 0 7 0 C5.02 0.99 5.02 0.99 3 2 C4.32 2.66 5.64 3.32 7 4 C3.37 4 -0.26 4 -4 4 C-3.67 3.34 -3.34 2.68 -3 2 C-2.01 2 -1.02 2 0 2 C0 1.34 0 0.68 0 0 Z " fill="#00B8FA" transform="translate(67,102)"/>
|
||||||
|
<path d="M0 0 C4 2 4 2 5.25 5.125 C5.4975 6.07375 5.745 7.0225 6 8 C5.34 8 4.68 8 4 8 C3.67 8.66 3.34 9.32 3 10 C2.01 6.7 1.02 3.4 0 0 Z " fill="#179244" transform="translate(94,39)"/>
|
||||||
|
<path d="M0 0 C6.625 -0.125 6.625 -0.125 10 1 C10 1.99 10 2.98 10 4 C8.72125 3.896875 7.4425 3.79375 6.125 3.6875 C5.04605469 3.60048828 5.04605469 3.60048828 3.9453125 3.51171875 C2 3 2 3 0 0 Z " fill="#079BC5" transform="translate(43,99)"/>
|
||||||
|
<path d="M0 0 C0 1.32 0 2.64 0 4 C-0.99 4.495 -0.99 4.495 -2 5 C-2.65772428 8.02934491 -2.65772428 8.02934491 -3 11 C-3.33 11 -3.66 11 -4 11 C-4 7.7 -4 4.4 -4 1 C-2 0 -2 0 0 0 Z " fill="#06A3BE" transform="translate(107,57)"/>
|
||||||
|
<path d="M0 0 C1.21386936 3.64160808 0.6674695 4.33165469 -0.9375 7.6875 C-1.31777344 8.49574219 -1.69804687 9.30398437 -2.08984375 10.13671875 C-2.39019531 10.75160156 -2.69054687 11.36648437 -3 12 C-3.33 12 -3.66 12 -4 12 C-4.61528766 6.92387682 -2.61658349 4.17017994 0 0 Z " fill="#E2332D" transform="translate(29,87)"/>
|
||||||
|
<path d="M0 0 C3.77545388 1.09068668 6.47043472 1.9083091 9 5 C5.26512223 4.5020163 3.18762365 4.12508244 0 2 C0 1.34 0 0.68 0 0 Z " fill="#D2323A" transform="translate(74,23)"/>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 9.3 KiB |
|
|
@ -0,0 +1,8 @@
|
||||||
|
#
|
||||||
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||||
|
#
|
||||||
|
|
||||||
|
from source_discourse_fetcher.run import run
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
run()
|
||||||
|
|
@ -0,0 +1,24 @@
|
||||||
|
data:
|
||||||
|
registries:
|
||||||
|
oss:
|
||||||
|
enabled: true
|
||||||
|
cloud:
|
||||||
|
enabled: false
|
||||||
|
connectorBuildOptions:
|
||||||
|
baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27
|
||||||
|
connectorSubtype: api
|
||||||
|
connectorType: source
|
||||||
|
definitionId: d12c5a88-5e78-452d-b957-eb4b2fd6e1dd
|
||||||
|
dockerImageTag: 0.1.0
|
||||||
|
dockerRepository: harbor.status.im/status-im/airbyte/source-discourse-fetcher
|
||||||
|
githubIssueLabel: source-discourse-fetcher
|
||||||
|
icon: discourse-fetcher.svg
|
||||||
|
license: MIT
|
||||||
|
name: Discourse Fetcher
|
||||||
|
releaseDate: TODO
|
||||||
|
supportLevel: community
|
||||||
|
releaseStage: alpha
|
||||||
|
documentationUrl: https://docs.airbyte.com/integrations/sources/discourse-fetcher
|
||||||
|
tags:
|
||||||
|
- language:python
|
||||||
|
metadataSpecVersion: "1.0"
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
-e .
|
||||||
|
|
@ -0,0 +1,5 @@
|
||||||
|
{
|
||||||
|
"url": "https://example.com",
|
||||||
|
"api-key": "SomeKey",
|
||||||
|
"api-username": "r0b0t"
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,32 @@
|
||||||
|
{
|
||||||
|
"streams": [
|
||||||
|
{
|
||||||
|
"stream": {
|
||||||
|
"name": "user",
|
||||||
|
"json_schema": {
|
||||||
|
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||||
|
"type": "object"
|
||||||
|
},
|
||||||
|
"supported_sync_modes": [
|
||||||
|
"full_refresh", "incremental"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"sync_mode": "incremental",
|
||||||
|
"destination_sync_mode": "overwrite"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"stream": {
|
||||||
|
"name": "post",
|
||||||
|
"json_schema": {
|
||||||
|
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||||
|
"type": "object"
|
||||||
|
},
|
||||||
|
"supported_sync_modes": [
|
||||||
|
"full_refresh", "incremental"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"sync_mode": "incremental",
|
||||||
|
"destination_sync_mode": "overwrite"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,35 @@
|
||||||
|
#
|
||||||
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||||
|
#
|
||||||
|
|
||||||
|
|
||||||
|
from setuptools import find_packages, setup
|
||||||
|
|
||||||
|
MAIN_REQUIREMENTS = [
|
||||||
|
"airbyte-cdk~=0.2",
|
||||||
|
]
|
||||||
|
|
||||||
|
TEST_REQUIREMENTS = [
|
||||||
|
"requests-mock~=1.9.3",
|
||||||
|
"pytest~=6.2",
|
||||||
|
"pytest-mock~=3.6.1",
|
||||||
|
"connector-acceptance-test",
|
||||||
|
]
|
||||||
|
|
||||||
|
setup(
|
||||||
|
name="source_discourse_fetcher",
|
||||||
|
description="Source implementation for Discourse Fetcher.",
|
||||||
|
author="Airbyte",
|
||||||
|
author_email="contact@airbyte.io",
|
||||||
|
packages=find_packages(),
|
||||||
|
install_requires=MAIN_REQUIREMENTS,
|
||||||
|
package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]},
|
||||||
|
extras_require={
|
||||||
|
"tests": TEST_REQUIREMENTS,
|
||||||
|
},
|
||||||
|
entry_points={
|
||||||
|
"console_scripts": [
|
||||||
|
"source-discourse-fetcher=source_discourse_fetcher.run:run",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
#
|
||||||
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||||
|
#
|
||||||
|
|
||||||
|
|
||||||
|
from .source import SourceDiscourseFetcher
|
||||||
|
|
||||||
|
__all__ = ["SourceDiscourseFetcher"]
|
||||||
|
|
@ -0,0 +1,13 @@
|
||||||
|
#
|
||||||
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||||
|
#
|
||||||
|
|
||||||
|
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from airbyte_cdk.entrypoint import launch
|
||||||
|
from .source import SourceDiscourseFetcher
|
||||||
|
|
||||||
|
def run():
|
||||||
|
source = SourceDiscourseFetcher()
|
||||||
|
launch(source, sys.argv[1:])
|
||||||
|
|
@ -0,0 +1,63 @@
|
||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"id": {
|
||||||
|
"type": ["null", "number"]
|
||||||
|
},
|
||||||
|
"name": {
|
||||||
|
"type": ["null", "string"]
|
||||||
|
},
|
||||||
|
"username": {
|
||||||
|
"type": ["null", "string"]
|
||||||
|
},
|
||||||
|
"created_at": {
|
||||||
|
"type": ["null", "string"]
|
||||||
|
},
|
||||||
|
"post_number": {
|
||||||
|
"type": ["null", "number"]
|
||||||
|
},
|
||||||
|
"post_type": {
|
||||||
|
"type": ["null", "number"]
|
||||||
|
},
|
||||||
|
"updated_at": {
|
||||||
|
"type": ["null", "string"]
|
||||||
|
},
|
||||||
|
"reply_count": {
|
||||||
|
"type": ["null", "number"]
|
||||||
|
},
|
||||||
|
"reply_to_post_number": {
|
||||||
|
"type": ["null", "string"]
|
||||||
|
},
|
||||||
|
"quote_count": {
|
||||||
|
"type": ["null", "number"]
|
||||||
|
},
|
||||||
|
"incoming_link_count": {
|
||||||
|
"type": ["null", "number"]
|
||||||
|
},
|
||||||
|
"reads": {
|
||||||
|
"type": ["null", "number"]
|
||||||
|
},
|
||||||
|
"readers_count": {
|
||||||
|
"type": ["null", "number"]
|
||||||
|
},
|
||||||
|
"score": {
|
||||||
|
"type": ["null", "number"]
|
||||||
|
},
|
||||||
|
"topic_id": {
|
||||||
|
"type": ["null", "number"]
|
||||||
|
},
|
||||||
|
"topic_slug": {
|
||||||
|
"type": ["null", "string"]
|
||||||
|
},
|
||||||
|
"topic_title": {
|
||||||
|
"type": ["null", "string"]
|
||||||
|
},
|
||||||
|
"topic_html_title": {
|
||||||
|
"type": ["null", "string"]
|
||||||
|
},
|
||||||
|
"category_id": {
|
||||||
|
"type": ["null", "number"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,45 @@
|
||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"id": {
|
||||||
|
"type": ["null", "number"]
|
||||||
|
},
|
||||||
|
"name": {
|
||||||
|
"type": ["null", "string"]
|
||||||
|
},
|
||||||
|
"username": {
|
||||||
|
"type": ["null", "string"]
|
||||||
|
},
|
||||||
|
"active": {
|
||||||
|
"type": ["null", "string"]
|
||||||
|
},
|
||||||
|
"created_at": {
|
||||||
|
"type": ["null", "string"]
|
||||||
|
},
|
||||||
|
"trust_level": {
|
||||||
|
"type": ["null", "number"]
|
||||||
|
},
|
||||||
|
"title": {
|
||||||
|
"type": ["null", "string"]
|
||||||
|
},
|
||||||
|
"time_read": {
|
||||||
|
"type": ["null", "number"]
|
||||||
|
},
|
||||||
|
"staged": {
|
||||||
|
"type": ["null", "string"]
|
||||||
|
},
|
||||||
|
"days_visited": {
|
||||||
|
"type": ["null", "number"]
|
||||||
|
},
|
||||||
|
"posts_read_count": {
|
||||||
|
"type": ["null", "number"]
|
||||||
|
},
|
||||||
|
"topics_entered": {
|
||||||
|
"type": ["null", "number"]
|
||||||
|
},
|
||||||
|
"post_count": {
|
||||||
|
"type": ["null", "number"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,110 @@
|
||||||
|
#
|
||||||
|
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
||||||
|
#
|
||||||
|
|
||||||
|
|
||||||
|
from abc import ABC
|
||||||
|
from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple
|
||||||
|
import logging
|
||||||
|
import requests
|
||||||
|
from airbyte_cdk.sources import AbstractSource
|
||||||
|
from airbyte_cdk.sources.streams import Stream
|
||||||
|
from airbyte_cdk.sources.streams.http import HttpStream
|
||||||
|
from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator
|
||||||
|
|
||||||
|
logger = logging.getLogger("airbyte")
|
||||||
|
|
||||||
|
USER_KEYS = [
|
||||||
|
"id","name","username","active","created_at","trust_level","title","time_read"
|
||||||
|
"staged","days_visited","posts_read_count","topics_entered","post_count"
|
||||||
|
]
|
||||||
|
|
||||||
|
POST_KEYS = [
|
||||||
|
"id","name","username","created_at","post_number","post_type","updated_at","reply_count"
|
||||||
|
"reply_to_post_number","quote_count","incoming_link_count","reads","score","topic_id"
|
||||||
|
"topic_slug","topic_title","topic_html_title","category_id"
|
||||||
|
]
|
||||||
|
|
||||||
|
class DiscourseStream(HttpStream):
|
||||||
|
|
||||||
|
url_base = ""
|
||||||
|
primary_key = None
|
||||||
|
|
||||||
|
def __init__(self, api_key: str, api_username: str, url: str, **kwargs):
|
||||||
|
super().__init__(**kwargs)
|
||||||
|
self.api_key = api_key
|
||||||
|
self.api_username = api_username
|
||||||
|
self.url= url
|
||||||
|
|
||||||
|
def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def request_headers(
|
||||||
|
self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None
|
||||||
|
) -> MutableMapping[str, Any]:
|
||||||
|
return { "Api-Key" : f"{self.api_key}", "Api-Username": f"{self.api_username}"}
|
||||||
|
|
||||||
|
class User(DiscourseStream):
|
||||||
|
primary_key="user_id"
|
||||||
|
|
||||||
|
def path(
|
||||||
|
self,
|
||||||
|
stream_state: Mapping[str, Any] = None,
|
||||||
|
stream_slice: Mapping[str, Any] = None,
|
||||||
|
next_page_token: Mapping[str, Any] = None
|
||||||
|
) -> str:
|
||||||
|
return f"{self.url}/admin/users/list/active.json"
|
||||||
|
|
||||||
|
def parse_response(
|
||||||
|
self,
|
||||||
|
response: requests.Response,
|
||||||
|
**kwargs
|
||||||
|
) -> Iterable[Mapping]:
|
||||||
|
data = response.json()
|
||||||
|
for elt in data:
|
||||||
|
logger.debug("Response %s", elt)
|
||||||
|
user = { key : elt.get(key) for key in USER_KEYS }
|
||||||
|
yield user
|
||||||
|
|
||||||
|
class Post(DiscourseStream):
|
||||||
|
primary_key="post_id"
|
||||||
|
|
||||||
|
def path(
|
||||||
|
self,
|
||||||
|
stream_state: Mapping[str, Any] = None,
|
||||||
|
stream_slice: Mapping[str, Any] = None,
|
||||||
|
next_page_token: Mapping[str, Any] = None
|
||||||
|
) -> str:
|
||||||
|
return f"{self.url}/posts.json"
|
||||||
|
|
||||||
|
def parse_response(
|
||||||
|
self,
|
||||||
|
response: requests.Response,
|
||||||
|
**kwargs
|
||||||
|
) -> Iterable[Mapping]:
|
||||||
|
data = response.json()
|
||||||
|
logger.debug("Response %s", data)
|
||||||
|
for elt in data.get("latest_posts"):
|
||||||
|
post = { key : elt.get(key) for key in POST_KEYS }
|
||||||
|
yield post
|
||||||
|
|
||||||
|
# Source
|
||||||
|
class SourceDiscourseFetcher(AbstractSource):
|
||||||
|
def check_connection(self, logger, config) -> Tuple[bool, any]:
|
||||||
|
return True, None
|
||||||
|
|
||||||
|
def streams(self, config: Mapping[str, Any]) -> List[Stream]:
|
||||||
|
logger.info("Configuring Stream fron %s", config["url"])
|
||||||
|
s = [
|
||||||
|
User(
|
||||||
|
api_key = config['api-key'],
|
||||||
|
api_username = config['api-username'],
|
||||||
|
url = config['url']
|
||||||
|
),
|
||||||
|
Post(
|
||||||
|
api_key = config['api-key'],
|
||||||
|
api_username = config['api-username'],
|
||||||
|
url = config['url']
|
||||||
|
)
|
||||||
|
]
|
||||||
|
return s
|
||||||
|
|
@ -0,0 +1,21 @@
|
||||||
|
documentationUrl: https://docsurl.com
|
||||||
|
connectionSpecification:
|
||||||
|
$schema: http://json-schema.org/draft-07/schema#
|
||||||
|
title: Discourse Fetcher Spec
|
||||||
|
type: object
|
||||||
|
required:
|
||||||
|
- url
|
||||||
|
- api-key
|
||||||
|
- api-username
|
||||||
|
properties:
|
||||||
|
url:
|
||||||
|
type: string
|
||||||
|
description: Discourse url
|
||||||
|
api-username:
|
||||||
|
type: string
|
||||||
|
description: Username to login
|
||||||
|
api-key:
|
||||||
|
type: string
|
||||||
|
description: API Key for Authentication
|
||||||
|
airbyte_secret: true
|
||||||
|
|
||||||
Loading…
Reference in New Issue