From 0068bb24b8dfbebec52cbb2055b3ab59ece59a3f Mon Sep 17 00:00:00 2001 From: Ivan Folgueira Bande Date: Thu, 24 Aug 2023 19:25:29 +0200 Subject: [PATCH] adding a Docker compose option that allows to compare postgres & sqlite metrics --- .gitignore | 3 + docker/README.md | 19 + docker/docker-compose.yml | 161 + docker/imgs/cmp_performance_grafana.png | Bin 0 -> 54858 bytes .../customizations/custom-logo.png | Bin 0 -> 10938 bytes .../customizations/custom-logo.svg | 3 + .../monitoring/configuration/dashboards.yaml | 9 + .../dashboards/waku-analysis.json | 3416 +++++++++++++++++ .../monitoring/configuration/datasources.yaml | 11 + .../configuration/grafana-plugins.env | 1 + docker/monitoring/configuration/grafana.ini | 51 + .../configuration/pg-exporter-queries.yml | 263 ++ .../configuration/postgres-exporter.yml | 9 + docker/monitoring/prometheus-config.yml | 18 + docker/msg_publisher.sh | 31 + docker/postgres_cfg/db.sql | 8 + docker/postgres_cfg/postgresql.conf | 813 ++++ docker/run_node_postgres.sh | 29 + docker/run_node_sqlite.sh | 29 + docker/store_query_producer.sh | 31 + postgres-docker-compose.yml | 10 - 21 files changed, 4905 insertions(+), 10 deletions(-) create mode 100644 .gitignore create mode 100644 docker/README.md create mode 100644 docker/docker-compose.yml create mode 100644 docker/imgs/cmp_performance_grafana.png create mode 100644 docker/monitoring/configuration/customizations/custom-logo.png create mode 100644 docker/monitoring/configuration/customizations/custom-logo.svg create mode 100644 docker/monitoring/configuration/dashboards.yaml create mode 100644 docker/monitoring/configuration/dashboards/waku-analysis.json create mode 100644 docker/monitoring/configuration/datasources.yaml create mode 100644 docker/monitoring/configuration/grafana-plugins.env create mode 100644 docker/monitoring/configuration/grafana.ini create mode 100644 docker/monitoring/configuration/pg-exporter-queries.yml create mode 100644 docker/monitoring/configuration/postgres-exporter.yml create mode 100644 docker/monitoring/prometheus-config.yml create mode 100644 docker/msg_publisher.sh create mode 100644 docker/postgres_cfg/db.sql create mode 100644 docker/postgres_cfg/postgresql.conf create mode 100644 docker/run_node_postgres.sh create mode 100644 docker/run_node_sqlite.sh create mode 100644 docker/store_query_producer.sh delete mode 100644 postgres-docker-compose.yml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d0bf39c --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +docker/data/postgres-data +docker/data/sqlite + diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..197a502 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,19 @@ + +## Summary + +The docker compose starts two nwaku nodes with the Store protocol mounted but +one is connected to SQLite and the other to Postgres. The performance can be +compared through grafana. + +## Details + +The two nodes are interconnected. + +Grafana dashboard helps to compare performance + +![Comparing performance](imgs/cmp_performance_grafana.png) + +Within the docker compose we define another entities that make the +nwaku nodes to send messages between them, and therefore, each received +message is stored. On the other hand, another container(s) perform _Store_ +requests. diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..4efbd43 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,161 @@ + +## Kindly run the docker compose command (`docker compose up`) from the same +## folder as this file. + +version: "3.7" +x-logging: &logging + logging: + driver: json-file + options: + max-size: 1000m + +# Environment variable definitions +x-pg-pass: &pg_pass ${POSTGRES_PASSWORD:-test123} +x-pg-user: &pg_user ${POSTGRES_USER:-postgres} + +x-pg-environment: &pg_env + environment: + POSTGRES_USER: *pg_user + POSTGRES_PASSWORD: *pg_pass + PGDATA: /var/lib/postgresql/data/pgdata + +x-pg-exporter-env: &pg_exp_env + environment: + POSTGRES_PASSWORD: *pg_pass + DATA_SOURCE_URI: postgres?sslmode=disable + DATA_SOURCE_USER: *pg_user + DATA_SOURCE_PASS: *pg_pass + PG_EXPORTER_EXTEND_QUERY_PATH: /etc/pgexporter/queries.yml + +services: + nwaku_postgres: + ## Waku node with Store mounted & Postgres + image: statusteam/nim-waku:v0.19.0 + restart: on-failure + ports: + - 8545:8545/tcp + - 8003:8003 + - 127.0.0.1:30303:30303 + <<: + - *logging + - *pg_env + volumes: + - ./run_node_postgres.sh:/opt/run_node_postgres.sh:Z + entrypoint: sh + command: + - /opt/run_node_postgres.sh + depends_on: + - postgres + + nwaku_sqlite: + ## Waku node with Store mounted & SQLite + image: statusteam/nim-waku:v0.19.0 + restart: on-failure + ports: + - 8546:8546/tcp + - 8004:8004 + - 127.0.0.1:30304:30304 + <<: + - *logging + volumes: + - ./data/sqlite:/data + - ./run_node_sqlite.sh:/opt/run_node_sqlite.sh:Z + entrypoint: sh + command: + - /opt/run_node_sqlite.sh + + msg_publisher: + ## Sends json-rpc 'post_waku_v2_relay_v1_message' messages infinitely + image: alpine:3.16 + restart: on-failure + logging: + driver: "none" + volumes: + - ./msg_publisher.sh:/opt/msg_publisher.sh:Z + entrypoint: sh + command: + - /opt/msg_publisher.sh + depends_on: + - nwaku_postgres + - nwaku_sqlite + + store_query_producer: + ## Sends json-rpc 'get_waku_v2_store_v1_messages' messages infinitely + image: alpine:3.16 + restart: on-failure + logging: + driver: "none" + volumes: + - ./store_query_producer.sh:/opt/store_query_producer.sh:Z + entrypoint: sh + command: + - /opt/store_query_producer.sh + depends_on: + - nwaku_postgres + - nwaku_sqlite + + prometheus: + image: docker.io/prom/prometheus:latest + volumes: + - ./monitoring/prometheus-config.yml:/etc/prometheus/prometheus.yml:Z + command: + - --config.file=/etc/prometheus/prometheus.yml + ports: + - 127.0.0.1:9090:9090 + restart: on-failure + depends_on: + - postgres-exporter + - nwaku_postgres + - nwaku_sqlite + + grafana: + image: docker.io/grafana/grafana:latest + env_file: + - ./monitoring/configuration/grafana-plugins.env + volumes: + - ./monitoring/configuration/grafana.ini:/etc/grafana/grafana.ini:Z + - ./monitoring/configuration/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml:Z + - ./monitoring/configuration/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:Z + - ./monitoring/configuration/dashboards:/var/lib/grafana/dashboards/:Z + - ./monitoring/configuration/customizations/custom-logo.svg:/usr/share/grafana/public/img/grafana_icon.svg:Z + - ./monitoring/configuration/customizations/custom-logo.svg:/usr/share/grafana/public/img/grafana_typelogo.svg:Z + - ./monitoring/configuration/customizations/custom-logo.png:/usr/share/grafana/public/img/fav32.png:Z + ports: + - 127.0.0.1:3000:3000 + restart: on-failure + depends_on: + - prometheus + + postgres: + # This service is used when the Waku node has the 'store' protocol enabled + # and the store-message-db-url is set to use Postgres + image: postgres:alpine3.18 + restart: on-failure + <<: *pg_env + ports: + - 5432:5432 + volumes: + - ./data/postgres-data:/var/lib/postgresql/data + - ./postgres_cfg/postgresql.conf:/etc/postgresql/postgresql.conf + - ./postgres_cfg/db.sql:/docker-entrypoint-initdb.d/db.sql + command: postgres -c config_file=/etc/postgresql/postgresql.conf + healthcheck: + test: ["CMD-SHELL", "pg_isready", "-d", "db_prod"] + interval: 30s + timeout: 60s + retries: 5 + start_period: 80s + + postgres-exporter: + # Service aimed to scrape information from Postgres and post it to Prometeus + image: quay.io/prometheuscommunity/postgres-exporter:v0.12.0 + restart: on-failure + <<: *pg_exp_env + volumes: + - ./monitoring/configuration/postgres-exporter.yml:/etc/pgexporter/postgres-exporter.yml + - ./monitoring/configuration/pg-exporter-queries.yml:/etc/pgexporter/queries.yml + command: + # Both the config file and 'DATA_SOURCE_NAME' should contain valid connection info + - --config.file=/etc/pgexporter/postgres-exporter.yml + depends_on: + - postgres diff --git a/docker/imgs/cmp_performance_grafana.png b/docker/imgs/cmp_performance_grafana.png new file mode 100644 index 0000000000000000000000000000000000000000..84f5638abdfcaca5733c4a2fba1ce7745331dd6e GIT binary patch literal 54858 zcmdqIXIPW#)-Hv}B+dEr1xU8IRkOT_dBNdW%GJob`?O=aP%i7MI4B~2jieFQ=QTzQE0dAkA_K`h((*`I8u8ZCHb2-}rMOc&m2FG;BJGn5J%u<#=|*nW zKtj^=d(*5qPbJ!{fKL8r=h~w*=4H;QU&BXzRl4TNL>KaPB2P9cpFH#QjA}2L<`vmM z@~P3^(2T6FUq0m?xV3_Fbu}v6tjE=CS@P8Gtr;yEUp8J&n=S|}a$Ou06iNt%hYnWo zb@Y^CwejbGS&()6#Ky(Vch37z>4$KOh0|U=de}|d`vWW>e2R2)^!sFGNPF?u4OzxO zM&tSaPj^;aBHz6)|5=r+`_}Nkv}EO5FW4rC1~~HsNT`-|G~R&P!M;T z=vw{J<$K<$r%AmeiLFE4V`_0o%tz64J?Dj|myA9`{TGyJf(wbqlExWe!G+k|)E8wj;$rU6sFToImB zhH`Rj&rExp5BnW3?e2G|c8w}2|?~C z$xuDqP!lEnHJCa*Ug8=s1%3v|6WDo7WcaHK`f=IYG(mz9xjo9)sOmo`6qVmrckGB` z_gTBOGq`w~|2du&9p&|Gm3V>9Oh>Vh9&4ZfnYsbkVx!}xu6Kr$tH3vN7$U#Hl6N<% zwkfRhUXr=G#!S)VqqIe}jZW9JwZ#=n-01(BzHo;75p8$v1Y`3-B?Z*M^dI4>X?M7$M>#zyv$_a1-*I@n_qs-f;{CWEZ4_abG_nZ>JDc6Bx zW-rQ4mJ=^kQ*V3~);+~<)5#IPcQxk$iUZ`d(dRpPwk@Vc=TKbB+*>_p!ZeezUKrdV zl%1d7_asp6jrB|KRPP#I6=^DzBuFvWO~5L+VHj4@Z;fD9qj*maVQA0-WclT({M&Td zr3=v9@e7cc;i6J7iWA%+wWpTMmy>3Chf=^lbC2#r#?SOU)Kx(lNNb>}z@WRC#+a0H zubp{Y$e7F0#0<5T*n(B&gnJVI(_y9^p`MFXK*K*f$}46YLpO8T5&Tb#YKR~)US?3e z*SS_0h8;Pd%P^umxWNdYn#)6tboMs72Egs4OWWg4ZSRMpK+1NGl!VY25c zi!~q5?4(ahVrclNAqT$CynYS^BkOwnqa2GG;OjM`@GGfB3PjzIkOKQkb2dB8Ot)Bt zUO78a)@E_3CtKt$*orUrw)c(54GK_$u1Z-!?=hdgQ`)f=9C87YWpw~7T4vM&)Z*SD zQKs2H$^RfvJB03L@mwCue90CI2S0ahlx5$Ou_(>RAo8^y7#^g6&>k3tLge+!oxMXs z&R)jmkCujv+O=FHDce(4qGZ6jOTp#=(~b*8zTSm7pDW;UK6L|?WGvtK-c?ENR||g8 zG0|z6h9DIoa_yffTSzC@%n4j!AC%@Xe@55PB_pi+S7L}sgTjaXZw0P9w_9!n7C-52 zPka=gSP0)GzZBEaM#)^b2R?_@ZlKrmeRum1w!-nu_089aux=>!9nBWbG4i#EhY1Hx zt5aw<@!i5u@$RUyP?*+S@*-sepe>r))b5xNpqAr(H6WV2Z4F3+qDyn z@@(3IWMnT@dz?YGa9yK1m=SF*Qq?kC(vVf6*Q&j=1|yOF5AVt{0Y>Vcu`!9F(-S zpGHF@DhvE*;5fG3o%xz?>z_ksThNL5w;R>LC{guQxURL|&y0>yfkl7s$hwZpR_Nsp znl#z5JdRon6CxhH!&SVn6MVn^ie~90$KeUqz)LG&X6I<1mCM+@gs_mY$n0$@MJmb7 zy^)XfEbOA(hNn}Q#)$Fk&!l7IQl;`zL?g837UArumF6hgG5^fb?~&O%TiPiR(LTAO$^yN{?z-oPVrkxWzI^H%c+Xe$h9TJLh*mwUhF1a|{#()nRr- zSHs%w3QPxXQB_K+ua+uOc~y=zgPVHfPv^=(+CsjBFh8e}XWw>PZ}u%;?fV(Y%B8Qr z>KWf~wfMmZ4>udwMkOAmPCJ3Yl@5L!V>%ogHFC5xIlVAagDMfVkx%3wpAQRRSY#17 z%Vz5f)seiC4@ScTjd(0R`ens6Fcd$i=WcsyUiI}7g{F&ba{{w-E_`Krx#rRz8Yb?c zn|PIYJa1ERom-t!y&DzMw5WmSg;xD+@ZcH9-nt^v>02PPyR<-d`=pYsw`Q!qh0d5k z@+bNnl977vOP7+9s6do7Q8_rZFwfM+r8eSF1{rg=K`#0i#md4pBLI|c$D!&6pqd%4 zO)lZ`!{rIGw%Ma^L0ZctE!O>=JxKmQrL_jB$3M2J*zI=KLq^j~Ol1O6O~7+nkh0#g zXu`J-qH))q&iAJK*oxsh4Gk$v6O1^em2 zmV%fhW(^B!ccbWy$$N~cjLUIys`P{TuJzfu8*;w)9=*a80oAe_N;C1SQ%C9V-VxNe zsKdh6u*f9)MzwH%fPucY4K3R~99=*w)Hu+KoPDXSBIF)zayt3Fic#KT4ox0Bj~T>~ zbr-oeCO<8~Yj~y4OPj-frAA1tI$hqTvArag=p0h9$6~@=AAnhTd>TG-Yow=D-azA;mZ$Q_i&I=?|E6X+ zFS}tBhemdWTK!M8l6IrPva%SX1$HTaM!;%i1ZD&kK1I`(T{a4iz$qs(LK=r24EnUo zH%VJqV|MOEuO!g$+qkey`%&k-^;F;8@DK{yu}KK3jAlXvb^2(22pHLD`pR0rx*R+R zdw2$Y9`^#xey)l?uLcrxRxEdAi!oZ%Ep{x{kk4~ooLg6{y`a46k7iqRyaHdjn_t2m zp_}>Uc^2e}J0Xu(mh)4-SaH48hx_Kzy8S&-|R* zzyJNR!=cI=t3+;Rq1Q6Fu6JV0%;cmOzwPP1-2X=e0+{|o)6&JONN7x{kddtPX-uEV;QW=0;v!xGy1pgG z%`|u8X~u5FK>kwN$drcDL+?!kW`i1?u4I@p^9@lC4JcF_W14SIcQc<`g)Y`$x=beh z_Kyqclo`VWRU5wvESJb+Z^?7(4@GRSvbF*TcQZ))X(Z z%BZpOGIV7@yWMdq=TyULl-)}3Ao#_BhjHTWT*(!1vQZSeKiZ;M(!Das-8g4DF@6b+ z)#eV@nWn+I31w(ghsruR#VQ7Xf7~O`Nq`Sn1Q!B5AI!#o0_n_zt#ItRZxyVZyP+V~ zF!tWnnKIR5HU5mXNzNQ5;}-dxU2g@wPW3Q#wsBEe>`!Gm>C1+Jvdzp;!;z^4Rsj#k z)IBEI1_rnX`lh#aXa2*)O*xycAv#wDD6}s} zXbiV4b)b%FD;4eUQMHf{6?7QQw>vCVI0Weq(iE5NnZ+usyS{LUQ;7u?2Wuj29Gc%v zJ@805>osH3r}zVWkRGz`4QEy;<@*heE~;pti5F8<-Dw+AZa-Y(+xPa)wN+3bTt+%^ z<*#jRe|eC~9_{7jp#WK8CgJx}9!wC2nty^A%pq;Re zjS3V}q+a|c=gS==ZyOfWTYVNa-|LCd6r-kB(f;}@6jP!$0E5libSZLM)mvvTuQ>EK z5-2#FT$?>BnXz{wnm^R|Yh_HBYM?1P`x~BT#=g+&?bk}IDbciwc(@~)JNJ}}=4`S= zr-pu?Ee~`CtU#?>$^M-8!3Rr&_?XJ>>yQ$2NW3O};CM{jse3!mB16-~LC4L2;)WfSL~ zEbA!6ptbjx|AK4V&%o!piu=k!}=V~?ONMmXI*3Hv8(JO+){@sPV5TU;*C&U zrL;i?cs`!W-wm3r5$Z%?V!NrYk`|+EA%gXyntx!obI(ZgHF2>b93C^tt5@j>kXhxj z5GGf`3KNSwB9~LCHwLnTXdfD4GB8kJt423o<35SMblcKK}8X0jwK631o@uUWyLYm5+ZP{2E z(e4V31_ThaFa(2(yPz8N-OAcai^?eVa>DObV@pk}3j0yz@add4B+G)8gwgxk{>v z_TV$?$LJ3WHk&=L70p;p_W3-moYaJT@B ziCUn~>=xr0aG-00+C(YN_FO2lQ#Jqi>WZA;o1a;g>l8IdqCm9GE1_0@5aGfuqM4U18ovOGP&~XZE5)akyWcigCMN4$!L&eh z{6V{ZaZ7jp0%{f<3nn16HRyD)+at2pK@%p2YS!jZ1r>Id_0Lpa$X!JB)%`v749=4G z+MPPJAaF+}Ynib`xkRd9j^1_n;-VF_U8{Zo>{hW^8eg;I>7|rBBURXWAWaxD_zhFP z-L>s%HH| z6zyT2nd~9g<78&g&S2S3PrL0^=vHb?90zhc-`$;Y-U{mXvEUTJ9$&1F*MoBqB?|@N zn0BUU5Vqux`oY1{M}$G<;#{}F**)PZzud>0m$8vUr?Cj!Es1T6u!M*E`_D6l@NNG9 zC8r0VGJS0KT>#=T<`FA=!pxw7!DL4zl~ev0-&D@Q_&D%t#E)JV{D4E$;fTN17~Oho zMeT%+WZq~12FGl$S`x_S)y&Rp3yR8d<&u@30S8Jw(CpyL9?A{Y^|TL^t+)zWwqL9I zSs3i*w^rX7aGgDl8hY7}7WQ`M$LwKf_c<|6joHSU$=sT;WZ(OWg7IV{)Gr*v8TRuy zmSBLBo4dSa)Zb?2o6PN6?^_o%wvd-O0vhx}c_!?e#@XHzs*W{VcsjFl=(nj=!fnrs z)gc?w*l6DLsoqG>ZfkIiwM|s$aKsw%8TVZL@u8-pFEcecy33^6I+kGBhD^0IpX{E# z$C$`7rI%FHc{6lso=vmAc{1GzpIbay+-sg3tCQoKS~%?XsK$9ijZptUbIi{*r_!Nj zp`tC?C3!xf!bjgR5dO^~7Tcpe<3vuAdS}PA6!qoe<#Xst zYz1jhL`ey5zb)>D^I9}4p-z|gdml@fw5aMaM})6cZ|E}z>MIN-W*UiUl-sN@pMgB9 za8h}Ksz1#mnT+XCt@qw}BF{&VPMn$=QLH4mVkwe!*k1o9UR>l%V#gkkm}4I^INMgjq`%6MMY*#l zBhhf%fV=+eV)(`Mp&t2B1}N8IzLS0$H5u7>&9g%>VPw&1Wakf6iUo`8&yoU#H?J;7dsGENi>LPLoPlEJI{+7hGE&^Tn4KvU-BKa5^}`_ zjI7bk3mM-s%l8v|xv~hjmW#2G)*9)x%dzPTHPmaLdmp!f5xk(MaLS=WSug-XZ=zWdZ!|{-zoN^wlzl<66Rhd7uZn3J$7%Ac#V1* z?CUb*kgX|}SuVb_7h7N9f)}yEz7&(@U#yDQ{4P2l{^ON)2tVBI9uxcQwBrTuo`R}u z7yUuNB`Li-T`!~|R%vN`;MMou3@`owoFzwm68s!ljbg~ig72LR@5^6?M#J-JhDEic zYiikNqhAya`1^G>YVE7t2cKrRfQu1qW8cHxB405%)UK*-S8Ac52^gmuf{-nxZNgszNvx z0dK3DQ(fAa?XFELhq(rNESOE4)g)QRKi{s!2x}vgA+8WHKOUjOMl@yi5dwd>W}>c_ z9D&4Ffs9sYOIGWPz}~m!*927U1Qh7WIh`Y-Xf+{G_CI6Xw{DGKOW-kA;O>HHhYSlv z?l!vqc-B-=M-9OiGV+TIEQ>QTb_4!xDrMJ@&t@5^+zu5l@EF1ESd6fiQ%nTyptwU0 z?FJ02^K`rGO}0ao;@%IUI@8r!ry=rKgOVG%V*TEJDRTusZeO9zG4m}KRR=C>>=(M^=1l23|n zz$vdF)pggmBQx=B3J3-ECp{JA?B=fOA#rs#VQ(f=`E9+bK0p2n?|{k3qC<;y^mH{N z3+HCjbUJ4^t2b{Hs5dY&t*qzNAa6CaBRJP%|8XrxS-*H;>)~65>q`YjHdlEhr8J;i zcjbPRl=MpCJ|x|w{m}5995ayG6Y^H2;IN<%kUr#M!W0wzo*?X0V7Y$f2xDgCjoTL( zgfx7W&&7Q7$o(9bXUqu>fg;UTMRVBHCIh7Thgpmq&!1ZuaqbEdqKvjxN0v$(?59zd6b*Nepyzk4Hde!z1gQnfb)TMKS=6Rye%?GUjVM-aL=ADC!(@Rg( z!)pp0>{{xEiIVTszN}HE;w6kD0)rM*4_J_%t9?oSe)C0|$(b8+h6VQ5uV?<8y+bM9 ztxTM_LrG}cu^f%bWw&pZGq1RSNob4a|D@PE&&1`yEN2k<8~@De9}=71jCd1-dLqGy z%3#bhsa|1y%s7v6FrwcSEd%KfcqLlPCwHX>6gy^DG71C_A`6U&@AU;qNxg691g2v7 z%7g0eP}>;@%6DiME)bu-xs!Gt`*Bl0#DCq>HiUyG*r<{3F_pM=%JPD(HrOZo^8rfX9ES~JcX4D8VUSNdVyM2SW0=;<~EF>8(ZPc^#JFajS|Q>-5l zdVR5ecfS~mxqwzyJINkp{|jMSGIJgT@!0G+-OO0GgId=G`uyUeaBEyRhY>T0%PS`a z#CsFh28}-=i!DdtSPvyV3Vb!Fyd@*3K@acFKnlT)=Ev1U~qp z)QniOqW9sSiQN*Ip1rQ@Z#~IlT-&kP|+6g9qr^inp$=E;c&Wb>IOe^5yCtOGC+AEd?ttlg7{`Ydw=Q-k5MF zY~-x&JP#hnzLBUs8-Eu_M?ciWMbegmIMqXQ=P~_Hlt%}joLjt}t?vH&d{>l_`4C+Z z7rzkCWWrS)76>LqYT;n{9T+th?tb4ZLg&lS?KNeSJir=EC-3n;-LTp@NGVqBE7pF+ z?irC%TzfMb`IPG$kcB2A%kO9mD;;!lulo9aqpyNo*I8^%yG#?GUpz_pR77nTZYb{z zLJJ;l)oVNXusSUklQS>$}8u(?7=@$&y*VjMDy+o@0@!Cl%lEp)3XX*FPMVii5hTS(MwehlE?I`2^ zaNTi4P?!$V7ypi(iwoPf7p;K)4o|jULvF>8%8@Bk9;e>FkGXDL>Z%+wC?7J}#OJ<&0~W8nT_8n*09qNlI$!^wNZnqqJxB)cDVidww_U2Fuk! zpm#rrcGj&u!e-Vt%lOZV&<1mcNymM?t!NsVb5*AAfn^wI7zvK7c0lTtGkBV z6UK=-@n6;_b@nJ6=xj1F$Fa5d`jpkpvO1KSbS5|Hic}!rq(h*<1JWQp?F`+?yI#NM zgJ$>Tt-I!qzgJ1^s$746ye?G7>%uREGYVLLy?;oo_dBRIA6Yv(d9iJNeDon>r^=o# zg-xsiTeJV_ynp`Q`lRYvU2i-gzdF`m(rJT5`EXqtXW>L@^5wn2;}ac85jvY`N1Yu1 z!%s=B^3U6EUxBIbxmGfOcbBC$oW9s#D=4Z=BS#HtG#2}r<_7c;*j!~gv2Xt%Rq59H zwatC)k2LHK1?1f?AEX{^;w%AK*n|xE6X41G>+c%oQ|E;EtrzrCCA@VaBxoUd`bWnq zl72gpw6tQlm~aHWSZ2c>{(*0URMy~TZSZIo4{FbFe%{+qUGs`$7Gnm z=8u_~;-9lKR((1f3N4~F>DGNy?ZWr#!HohG`S3JOytod#f@|o zG5yWeVTcPCFBaHwuyecF)v5P5@0f|{FK+)P(`(c0R~;My5d9YeO0nGJ%6 zynhS(P?fFM)y0jisIIOq$j`6(`aNPovOA`&(QCGdnVnTQ3%RJvv6IY&T*eIy=yz#f z9A0ypr!|Dx2zt|^&~b17R#f#3wF)ODk_df-5ZbF(MQGn0N&GGOV<7~D`jQjBiehKk z14t`%&-mVtaf*e}`gjdD+i0fhnwpN5+AOEbhkB;AfbPU`8)WBnP_y02h?tS9E0FNk zbe-u;q%{ohJ}I~M5#n7DuS`|ocRCy`}aztc7y+Ky&N z-Cfq)GTDj@4TUr8uTKtc;)Qm1W|PohPq%*C5x(a!Ot6m`4=bE!wRFC&Jw950jT0g3 z>%X%+F76|25E@^XW%=0xZa3tnFjB9$!s$`pUkb++s?9|s`0d*7wxiKFXu=#V+8E&V zz-Ba@Xp40%H(iQBrEgKu5URKWpIl@z5B$XvM^jR4zj?XMAd_fG0Ln_vD5!BbP><*@y~d)XZET#Ddq%Zeas9J^?VNwxg6UDRR%53yRzg1rt z$cle$sf0=7%Oe?|xn`P~g?W8d%ypvr@VO*j5xtclaa15j_!HZtkw zCrVE*x9+xRPwdyYm>8@1GFfnqZPahS?ONG?fe#{tv@N?*V5MUd6JI&jRHss{VkQk& z{>HqtuIh;srA?cNKH<=`aa%($>CA{j2;NRxkKU@Jh&pQrG3tIoM`(cJ3#wuQ`_u*b00t zHby-DpxV`2&KAKcC1s-cJX2dw?`g39@%Fwtf&U5s3Ch9fg^`3aUP2a+>P?-U$^;e% z62tBv!QMm7UqWjTs+S3bTuy$U2(Qkv{JH@G%vpeLERK(7Y!lMq+8n{{cA!H9=pwb; zfTIqn)XUtrf9a3kJHfn^64Hw)-Ap?=$d~ah>x9u1$?^+}x%|`^eK&kI%+=W$*U6Hm z0U!jX+BwIIXVb`T3`L+t|Esw<(h+0hQ72~`xI4|)_Kpq%F*iSW^9rgcb0U-0ltm?> zpsr55er>R0jX>zbv_-HonNk{sO|O4E#bALI1>#G zjfCiA8@L5rq?7n94T9^J2DUCq%wI5&ja0q-)`?{(N4^E@)=iPZyYiQH59*fco54Rf zHR=gnHUeM$D7RkaDqZdubKp{c)Oe%Fr~%wIW(#v|=<4cna&tsgeW){ZdU4-tmp560 ze9Tf&=7c;PaRU;8Ii?JKR}q}t>;UH|t%sK#y0|0ON7f!mY zAX`BBP23khIOoi|{Ct|r1S>Bu?|C(=a-|IcK2QRt!n&QMm8;Qp!V+Dh!U<2Ag*5gI?d_D>zE z3Rn-8PTb{o%~MHfjb`#qt&=@{?ozR9VRCYjyUjjeN4Ec$y1r?aj&nvVA`kD&0|oGTx@BuCR!{1k!>^6`kK+9fl9KAiIl=#KMSu(rfFB z*#pjLMXO%Pf_D7oP9M*1FlGa67F_FxgfLmM66LY<S+VqnI zr>{)U&f0*K$^k-3>;Cy@!)s4SOD+jeLqj;d*73Tu{DLej28gs&&oIk>RRwFC|6UdH z&*lN1Aw$}(rSH|lKrfdS>wot#lfX&=2yM8I7srAT(jK2}S}l*4peGIG;WfEmw8zF< zHRA!7QH9!}@>tfVPX>!OCjsi7s4AYb+^`IDo>d?iFVzmBd~NP_9iKL*{AKGf&zHwJ zIXL?K`?fG*_7iVl0DAY8kDvFlHm(XHub01k`EuBO>7(v6AN}es)cS>B3>FeQump3n zlZZ)8&8c3ugxA##|GN%vgE3c;icM)6*T{LiOJu1d|VGYzUJSyeqo%Moh_pn&(+&&9!mL-Zh7as#@Muan5grNQdnto zbC~W_`u{=>nYlB9>QH|tr8%p5nVVP9T}ed%!+?C0|y44*(9Y}0Ej++0wP~=cj!_gq;43_+K7k@BoUT6 z6sa0yP*n)2AwH5Pg`T-qt!(zUAWHW z*b?S_^gS-U>&&3F^?Lmed`&LSd0nC#laIIzZ{{p56#~aFk9u1@mJYI|(eaeEpB^BZ zQ1r(Ji4xzjJVj_HgM~h4U=qKpZP=EGs3>GJ)S0mQ)pc+~HZ$nJ{*6V$ebw6$s+!i; z)~3pgq}ICM9-9R+V08T!h&BsEJMagf5~u3DI9Atx2WbRfHf|S-GFT+;ZdA)sD}IpN zU%ED6E9iT$^1D9gnT0;Fk6-oNw<^F_=7_Q$?ZtyhJ7he1AqDcZs3v zQ&VP8Ccy9G*sB%+U#XSrh%eGOJUCr45`6U0h2fQBPx0jlU5d7ZuwSO2-SBIpmh5S$4-*gJKbJ^XL7=)~Kc=(fw07 zxqf*<#zq-|p<|eh@$6GAo}Nk6;*WT~UBrF-u+`ew_@UqT$h0j5 zAEOUA_g>%l(%d@7nbBU=^Xa~Z)m-brO3g(^IW!i*Y#v{wog5u|viQs6L#KwnR`>546zP!s1!(zJgEg>ox2LD+an-#V_@ONmPwySAaK2;-Y;jR=83MuY zeqb9LA8!(%dK7Nj&N;Ep(`7_HVV>(!%eR?*HcJ&06xb{2MV$2H!*3Svc@B7oBFFJL zUW=(XEiFS%L~!+>g zrS?M323Jzd3E`Or+#1Vx_U0F!)h^`47=i!S)JvUQz-$COA>zgLH|T%Dzpj$2@bEaC z8IgSGe>qXZj#yGl=&tT{7WKHVsX=(XHvZxC0waIX;5NgEH=Frv1w2Xdf7kU^BxVK` zGS24z$s7MI@YUMuq5S;a;HoL3#>DKQhI&-apc^p6KnW^t#+7ggt~fPUN?dvfwGW1t zyQA$BB|2#zEE$iyXbPis9b*Jkwz%Hg%>-+hmMUPd^&U`Bc23qv{IE!DY-!yQE9a*= zAx32b`cK(?c>{GTzq zKbP;Ac1y?U7T*>yi~g$zL`M7iUp%0uW%PK;)*9VHU#-J5y1K=gFe}fZdms)3l}6lG ztUwx{E$NWESyZtlJ3FI=-HFd?sx>X?cULp0xx+QwHm4o*k3467ljHRxs}&+SLcNoG zj6FY^1_X@h+_s(&Rt^mUB? zeqx~K1y7$O1w}>tPY?;;$Z)jBT<+7dBF4>X?2)Hui;b_{_D63*WBatI;~iC&b=8Fc z3R3}}@Ac4ZNniV@W{rGRDWZ&EvU2*N*duDiSGyaR3riXQk71k?HSJaISG&_2@gZDvgtsZtDU* zAR-h#kK@7sTx+9yN{WpOFShTyDUO6p=L3Y%oOdK6GxKx%YBO_l&5s~DY(Z?6VM?mD zo|>ASe{-{}ll`P&OKG9Pfkh~j^@@L=%Up|edxu!Db571+VPWCgYyn!2jfY2fv7bua z=fAj~HD&qrF$e@A#i)R?X?nU>ph^R0yUFj9&UNKKYYHi7Se0K@#*J7MCw-H|oR)?A z?eB{|fIO)c6dtNv$w(I4vB9~s#SRr1&+j%hGGgkzRRN>K0$Lcj`|unEeGC)5xXZ)+ zfL!e%lZfT|m@?-f?QgLET|$)Ot0paY^S=c_H>1poI>QUZq4kA6A~h+FOzVC4dn$)S z9^+*T1t@BdS6!1u;*KhMb8U08XA0CuQZx{Bb9sOZ5v+eVXar7YU!|V4Ow0o84$>XH z3{UF&J3de5eN2nUp17d`)~DLU>vwlMyw=_|z@)u-@P}h70<$@@fKyVsfo^EnRx%!? zM_M&{L$lG6s@;G-GlOOo6a{#kDl8;-${A8uv;8_nvjHZ%Xc%iA;P>u7x&aIRcS$-w zo^(vv$euv|??7a>QGEUpjFEaLd&%((I5)}M1_YKLTL!`8x6(xY#B~)9$L<`-01$0d z+uKD0+9W-_oqafvedZ$I4EdpiI9x0e?3Xe6!G7ZTfuUJ3slGem+@uCGc24H=y}og0 zn9m=F8?)8<((&WMAKMK{gi4x?*+@2a_OjNcXK?^{fizqCJ<_wXvMd^ebr~HD^OQ}P zAag5nR+|^sDc837V~wZzv`Ed8DW8zwPsNHDQBhHLu~~V$y{YsvFq&O}AU4e)m22W! z2ZEA95^E}$gG+IL3y%-_N9v1ozI?U*9H3c#M^Gg+b1b@~B;7dNLz!{UwCLbqLR>^7 zkqPu)L$@}^+0Tr|6CMgp1-_rjIpR3%C2<1Y9__aFB~4P=oL8o%r@4eGE4u`b|AO5?rTRpDCvvywuQa=%Xv)?*82D0P*_y zd!mxiTvwwz2zu{0IHr6tEhBy=(B^x$_tf~f+M?tUIE9^1!CM2>jr!V){Y8Y%LK>v~ zJ|s3K<_Yva#>qe+Xea_mja(?eXH``_>9R|)8EL>mwAX|Tm_Xi#$+^-}3>wCxj;S16 zSd{N!Y-POav@l{o6F=z1Dki4rJd4)x^xP5H#gO*prOAmW4Cbzt3ebe!ym@m2W0Hnd zm6~S#(HFr{$@t$>u;t@G;bD5cSVU;e;Gyt=9CfVnqf_`0Vp0RA0cL5?!hXJp z6WJ+%EKdf5>;V}#PTjX|toZ?d^P7mmwa^AX<_nHx{LZ^Vr;&i!%f zwc}HZae|Myk9Z1?W!F}xH)@P${YPhtx}03lq}%3-UR!5pOuViRWjgo%6JO#b3}(x+ z?KucjHXTYYi(lE<-EGJ57(7TxWTR>la}5Z9Utd9R-jR21?FHg}0XM(hR{ZmlEmR+@ zSy%=x>S72wEigrhZ8j|zj?5Ahp<3MCk$le9vcJQ(2i77}~8Jkkg$8WiONx*JU zY4Q$>^$N;JK}o3-+YjNYz9S{cjgr~a9E|#p-A3j(t4mYAH^l&v$Y4%j?0l^~z%>4H ziJn))Y|iy-lGE|`NM(-3=H=zp)%$7{r{rZ9uBhG+dH%j2P+NDpJyu{makku!SP9Mc z^>*hFX4Rt~lqg)A=Ezr^ZK3?fKk|X6G}huJ;UQ`*mjPk`q7GfMwr+2I>UoIOGt?6!Gy>fhP)ynVHtl4sWu-_EtNW$}XJNF9_0_p6odLpLZ;BI_x4jc^_p-qYxu$7-VFW)YnpuuSow62X!dDkbL1U z#y;M@fw$jzTmSe-elR96ZkI(^KzCo%z z&UCztYHrnEexF?rFu??DRrZb6*8-xnrN)AU5G=_VALun1`{)#b%XMmlMHQgp5)xFa zTaLS4>t_i|`w4>k?)7(!8F^IEnZE9PvV#ZGgq&Xa&K90TYE(P4}8M$_#uhH z2HvPt%kQ;lY(aI5CvM`M zn<8^tD>EH{(6gu~f}@ic%V5lBWd-`}$>iSCO_m z&*jyL-Y67mjnJWU$2^cTN(pJPWT{#KyPRWgAUGGHS7-Jih=M-n!1o;6?c0wK7V>OtZ5lu6k8HTg z2{@St$m>1DijRRI>ua1EEkBTu`QhsCcL=XIwWSNzR?ScU=e=rO`x+P&G_U2d;X0%y zoz(OI@@qwIOj!ZO=V)<`E=Q|AVh=qVb0FLUxVY;#3uWdKxysM>E&` z<1{q8J5udi(zP7q;=yF$?W2Hz;qS*j?op-0#6-5*k(sLf@zbfU6H|`#Vs0`}0QYn8 zVinZiRi8d(R!$eyT@)--Z`n+Zjj`BM{at-zE3ZsCC=VZi~*f-tFAtfBIB)XFCSxyCr!Gw17pVPfSc; z{GY1Ui}<}CMB-T~C?cNw`y2s>^9gWlIL)iRu-N*vPcgqVzz3jHhUpO0Je^hYASL{! zMBz~K*auy?_dS3G1Nw6*CJy}tIZwmedj<*y3FOK@k z`m?Xl^IcBU;@JA3z9dOgg>Zaw!@%73VR-6L`FgM&1Y3sGDKw0OQ~XcCVqLo;Jo~vv zp8bm#RFl4OvM1tX@ykd`$B{10A1K;{XzReS15E&2x0boTr#|-|RX3$9!5oB%w13d( z>5zQR<;UaKexnWO(rJxw(+8;Tt5NND{Qmb&QHG zcOT3!1U6XNwtBLEI|f)RV}FXA|KRU~R8 zQYXDNvkTqPZocKU48Bw6J-vx@;X?-b%|`Avvs*)<viYXL+|X>MP}$9LeseQ3 zYM3f%3m&$eq7%HCidl-A8AK9WZrYtb&0)QZsRma*3lptaDa?_%B-BN2ZzWzauJeie?%ZRb-@S&KSE!Sqt@yKa^fra6&n6R{T z6XieDK#WT1CDV#HtBux%4m3^`s#U3SnqJiBmOSzFxlOqJh?nOk6(waMpQQ?X>?I4( zoobS}by6jC&6o&PPaw1iAfd4Yk&T;MW&hkJCV8?Y-)Ja&?dQ){#E}%eTs)DY zvc?=1;fQ@FCB=B(Ox#^L$hfm=2O6y!kWR-AWLgN?&}L{>JvQmiSP05<(;P8SK3)3_ z;NVg#J@2AfC~4eP0D>&)@zXLeP{+Ca;^pGZ_Sn~y*RB;gUb_>Mn5geVag?-s`cDk5 zm%)?H`^&{Dd2d!C{963J*`;<|bNf(&_L4ktra^Q$Ab3ZP1?BA=uke-slF10ywWtf{tR-X%+< zO2qiYR_dy0gJut=KEx;#C!439EK=@E5Vk7WnAN3gHh#BkIvW-n zmzF!(5&>%0S7VTJ)G7jRqI*z|S;$R9T9Q1%(w2dprsK3d2iu0o?6sO5kSJF))TRH)Y1;Hl7i3fv&xKSK~5z zuU0~@f?cwUK_zWZMc-9~+qnOhRL+}o!rsE3Ya$MsK8CLm!RoAknZh+jnv$4LkS+!} zRPn)q8%r^fn%UgfzrFQ}sTNmZ4ZCh};VHCup50JGEE9gH)}ZlG<3jiHvsfOJM@FJj zxd1x^p4L7g`nYPn5p|op15zl{GgDR6lsnsTk)&iV;U{T8K;ontcm8MI*1W7i&9hZ# zAg!cJ%u zoGa5JhqcQ$lKA~4alUrYI$v>7FO@Zu!E;GHD?m^f1eDcw_ zTj}e9%;jz)%n)hS9Qg3EFtgBxrqf-VQKq0r@*SaHx* zPoVA1`IjizQo{Jr$Vm`PHZDj~f9CnsVu_9KStDAduS-xvyJ3>^F)<=q{s?$p2>uB?Uat{8J8R%7D^%ClgL@$GAvS(t@8{j{^x!vdtcl@8 zXEv{0*8}zCbzy7BZ7`+2mdwls8(mY5UT)(T3yf;RNeBNb$2=9?YL!o&dPTeW`40ie zcNuHz`P~->=LXf^`jW;K=(8ItR$aIG>dZI$N000k3W0LhNor|mwg`s7F-wL zi6pm*G=?Al4|ne!)>OK+4?8;QSde1@lsZZWfw2J6RYaPUP^D%p2!!4R45*ASf`Ei7 zT{?si0Yfh~q=P^bY6#K-2_2FEA@DsxXU?4W%z5AUcYWXQpU>+;vdLz%v-Wz{v(~-t zwboZz9J>4G`H^Zp;OY6(c6pQTY(P$J;bSg`y7;d@*^SXS3 zQ#z-`eCfe2m=;?ZdIBj%%clwj+}iD=c1NBa+&Ul_Hwxbe4siZN&T+q~O$7|S%37uT~x z&j&U;c9MkU2W^_0=kp$wClZ$|bbO3G{Z0PN)&l>u@qww0eq@*My3TNr55%kFV(et_d zw}O8Qke$`-*I;|~><{><^uFbNlnL+|TM~sYKoZAY%7&3`wz%fOkFCbAx2B2X<94WV z87I_Ldv;NS(Pg_f@5jWlDteGAgE} zl2F;2?CrzDnxmr@mqx&YA#XbPscg2KWaSF@Dk}}a0>yVNR&qIJ{9DL3R%^^!Bkyp3 zAR%MD8Dn!jOh;3+Z>%j3GxXkYQv?^b7AmjZ`=6XVxb%PL4leo=rkQCS^mNsE&J9s^xMPlt zcg$1|W1k%G9eWpqLs`j~dq`;iC3f7GaVCouI}YfE_?-P(cdl8#7sQU4ktukjd3O-t zVDt?y*5v{yjNEVk5)`TAb#!!$2kurc>=6X+bKL&;%1^*B?q5-I@Sy!;#kAB&*ak7l z=jIi0{)aue`x?DEJU0$?ZLM!?O0*(YpCP{cgcX&8&Oq$2++CL6A#W&9`#uVy$&R^KOM@JxN zF=?veWU9Cs@@dFZFXtma{bXXjRjoWcJTC{LpewEbAy(SX0Aaigx>!urW-|t0xdnxb z-M~>Tuxsa13Lhe_QmOH1XvN(tmyB1I)1!o3TND$B!S(PSl~IHT&+6XeY+fnpmg%Ft`3 zX)3=PeMk#htWw;?^{bPo3Wp`H_zP_uQFVpE$o ziA9A(iT}dG^!glNF9%iHBxS`qfQ3z*s?sQQ^+b6eTv!Qg=6jh7JS_c3)^$c9FvY;r z8)ZTw60!?r-Ov#kp161)!vUTB3GmmZurdOFEm%26pf}##mJl@H>d>q`HXCDhs|$- zuc+aO#7C3sJB7v2xkrpfOjnBt(jvBBo(6Z~D(0t2eU zYj0fLg2HUiT(}9)(wUA|ebqxl{|7+Wi9qGudmdN+wx8yPSQ)sxZa>JL*irIz?E`3Y zp=mqbveeU}q~d?+q(x=X=#Sv=DGG^F#q^R~_1Lty$TB+5L~{%2G9`?lf=ag|FNK&c zeu!92M56#)nQE$K%G{`Ihn;lt0`AtSiN!(#GNWc|RV5$7po;nHB*U*Kx;`tVQ3wtPLJMHy|Aj1TiPhpAJkBIwzE1Q$bcy| z*ecjcTpJ7D%&p5W-PQq_&EZ4(1_lNy>sJF^MnE~kn!$sL@zrRZ|ISD`%Pduj42ZHL624(W)!IfxN7D^Sl&xyrdSh%9z(6({YgFSocV73_)9i8L$7l)@J`&+mD5zpC9TrZ##v@z>>34~AC4uB3m=hvG$Te` zRMI+h=uon(63BOPs{a=gddwAI{Qzjn-k}*inLnR!gE~jRc`z6P6%>X*NxiP$G9sFH zN6km#X0|Qg1K-TuTgd;zhQD|DYuIf_dssbf4&SwLyXReXbp?P_qyyZKN2PB^!L;O5 zjzI8GVZj;Wxa4H=+@0MG!TNjJp{nF!>5X|kdEA#bT()>@K`WVnP62tcV$wl&KiZq! z3rc^lX|lF@l{EIKz(o^|@kpaB5|QnRGP5831q!GHdeQY~hpUGUfE+@H(>;v)_KD6E z#&b(dP4l6D{peM4(Xc0`msU65+sSmF^Wn8cCUm7mNEr#_)hWE5Db4dIR`S%ra*`LXi&>P0QG<63H*E3 z`48Obzh>@tGtKRvM6bXlHzmYd5%;XZaT023L~9opxs-p+I_7icu|)G2@jdBO(%C+^ zPwWuh1o;Tg^aLK>lDST}toiW2$FIP#oK|K~fjj3$+bhrV=6+G|);4lCe6y_De(paZ z{<*SE5=dD;rimqlRr447_tc4m8*COLB3FVHwj;P@Yb-#gND=K7I=czx4O!7=^L|Oi z|CBRaUAFsED1X38-%^<^W!~ALpG^NgcX8j*_7x!CH$vSE^N*$B3hvz7*w}DdDh~#G zDsdJ>8r4G$iW1lPl)PhvK(4neKUZ8Z6;9h{W^QrPtZm#>Jn7g@hdNa|PDK`dw#9tp3{;{|@7C za-=w!WTh(;=+9X=v1EzyUa)qK*}TdLg%U4P;qpz-{sRW-X6@jRq_3d~tz?tUQTPLl z{5SFmP%6n=K-dWdK7PFG-w?}xKS|h{?+@7RTass&#`688h4R0`W&iu5Iw5xZ=+`C> z$F_aBFFs64kl8WGfNvMxW5g0N@Pfg!A_4*eF2queOZIwwQ|*a|37XZO=`4COKWrFs zW_FeZ@T*#0HrR!eJnET%phIrDlkFY2TI+pb`6(;z=ToZS*3WVXmuv=n6o;j9bt)-v z!59V$#|LA~BGhb>7O${;pe}Yb32jT}R1vQW;jE_kBCmv5aRFC)u(p)dNH(wz6e2Rg zX(QZVwXU(UvNAx80JRaIyzW7>^}qgjP*@xHp$HJM%&}FFlTDvL!+JzlG4N);ji``& zMgetBpznf%ylL?ma9Qa6AxxK-yk2t#weEO(daVZCTMdrU#6B*xD1iU^OKL$UeIy1W6e z(g48~(`H)jf^)9@L8j?hQ+4mjma}@1$;HI3^w#odkYy~iKiH7@=FKmBs-9ZL*`ub; za~9ebje();JcXYllV?V%Z+C8xj?NEJjR9`wt@8J#7xqh}T35WMH+`b4s(YTtUr#tb ziXz;R9eO#0yO$THb;-Jr_Bu2tw62YV%wOnTC*t|+yza*fvpN`dSs~e+8*H;B>p$lI z(9IqMVIL*x2htbXN|`az(Ug)}OG;DXLFCS3Q4RoWs?^YHUaQuqEXTp{1*j+8H2fE? z3><)KzNot>O%7^aV*~~*x!cruYcmh5eTk!sIIJH*<$##KyDb^RIaEehGe4WKx** zLpQBgGCQo0N(eS%_@=GQWfRoXutm5G*+-v#WeZ%kz`%Me=lN@BPe0(i;#Ru9P9@p` z6)DsDTppMP+<7dpF*h?V+67STm8c^5oov}uvK&X6fTU8(GHkz4U zFVL5_Q)2#!GBGtf>nH}Ki}S1-^&e|fur(d=#}`11V~NjMF}$rGZ-1>>jd~PPl!$R-TAT#}HjB#XWHURj1C=#cNeTjXEpkP&UzsklS zSvQ*V{fF3>3lWF(_f@zuZXS=x_SxRr;2jQD@VK>;M2i}qn25Wp*HdXFhxA1$BFaVw z*T1G})J*NT*1HOT6@p7ZK_wLivsOtH7%!3;uEf+Ffd!Ck=Z<^nM3!eg)3m_wnqMU> zo1N5&z`<}|1;f^GpM3(B-q#(mu@~#&9p2hdWqNI&7Z$ZTB_NOwxJ%}>6&762h@^oA z!pBXA9_6Oy=GjLd{NZcMk1%fZH$wClNSj)iTLTlljagC$DQI$sTO$nQa|_K-LkLfM z1^WeW_?5+dxw(UUbapgE_sI|O)aCJDkXm%w*{){=Sp0Q6XSI1_jTU%;{Ls%LrYQh4 z($UxN-9$H-)CaH$aim_ODIb9i7Z19yew-}R_>$n-UqeO4r#eH*t&$T>tT#M;qgZ*I z%5y}IIeGx;i6rTvsR!xRgG!29aIgi`#X2Q>ho^d1FILOWyEIRNvh=m__HGJbT_{ux zOGEKFencz~$o9DY(uII!%*EwefrtIn2NZ9vxHZG>}1G2N2pZlo@daZft7mk zV-n;qC9xIGg!9^fOSZX5L3>L}(`w*zqpR5qwUsCkQ%e9Q7K~66IW3bUsU02>jROqh z-YMO}{R?PdUDF~}vM29HWOu$-S3`oFeem2hu8{GLQjoy5gT{yp3aWqoYO#2l36G+Z z@qO>r`-3j<0+t0CUnttD;>ofFwJ4%iTL6~}cvjf03Z=CsG6%x*!K;{<*BNwDQ047v z&nclP>mdn2!H>%eR#B}Z&65d>a-^z7RA(}!rRO8!aM#a)={^*>D%ar}AMVM3YhK?# zLIZ$Ht{>#DSLo_mZi!I`F}M5H+@BBEpUOvn@+98#ZFP)OuF|BKf=qj_?=5~+PqFU# z#&iV~v^zB+4s)vgUbJ`LjxTd;eFMPPln>8-)b4GanE;89k>&C^g}nTH3bU|3JP&Lk zhmVyM^))__YDB1XKP@@P`n;QLP{p7G1frN2#4!CT3!}`G)(L28d*t#8(^aJ-7$XTK z83vAZwjY#%Q7<^#3Am0Q2X~G5ecSJc@h}0H7EJ3ocW>H-_ZE9U*Kb566Mge(3&SV? zZz@&D*segH(3#W2XzJL$c?{q&0$QEJFwG5PZ-7_q=aR9-W<`@YV8Nj54(NYZPZtc+ z2H0~De#_?)I#EOyrrEwtwfx{)f#&fNre@vwn`AP*`$0E&f-DY>xsDEEq5*iTZHeei zz$nvRWBeM(r~DRr zqIuGEW9lJ^PYRdSL*fKR?HV`K$}608qm!FBl27sT(qS>1+=$=gM3q{vb^;KF-Wk# z8#EKU9DpTKLH;LdQMonU{#yS&kiFN@X?tbA>!bH~mkW7ylw9Ud6zQI?TM>`w*G0w3(LOimvCa<`?cLw5de6Oa4<3UIFfud3$)w8PR zwf|%uHp>O??6GNhqZUjQ56vg`HVTGHSj5%EnPeBvG1iTP0vy17Ir#Cz}mg=?ul3VlR4UshiW+EdR}^#B7D2 zX|N9;muV;hjGA1)P)j2*kJRwrHb#N&5$(&XkM{)U=2Qix3%|POo4Z_f+256GExD-J zTILSFY|a_%%W1H40kHxZTm1C^2)qO^sqU@?>!m0poT_@boW&`#p;bu;2y2U%+~R7e zfDKx*s2PA*42VeNRSsPr0}8|SVeo}~{h|P>T5CUjXn(EtM2gSWr)6b^r?7};@fllP zfsXV%R|8+XS2w*cS2xjnW%9Z5@V4g`mn;nX$pf697nM4qpeZ9GL-4Ve1j|M2(j{Yi zutJgBtJTT!Zjc$(y83#K+l6Jee3!0A`vu=dN1l0|IreMXi!LSIsMhnrHHD#{>o^Pj zh$)###-e<~ZbBhXhgduZHXgZFj|>*|a`|NQ_R_?8*RGb*%++^%UUh9I>Cb%Xfvq8l z#XBeDZX^|(mPc)`<-2Pe0avJDvoUdI;Fq7i02%3`8XT*QXR{h3}MLRS3%t{Y9dl-s>@^IGltwB_!9V&MQ`4~$bEs28-|iF z_!=cr64(nGkNHCkyx}?=5+;UV!SgSy{~pDwFymlsm;fr4xfRCUSoX$0Fc4`MfrsJ7 z`CSHAFhWtSby2N!Stqi#f7W2mwDcjbH&~`es9#MGZ`~EVQfo(+hBg@^_?iL4fYYD{!afIr#1p? z_O?8HuLU*PVbo+(bI-MyFipJCW0e}J$;ShFYtaRMz$+DZa}&_K7PGr`>@WjD6n$oj zZ?f?KIubM_gNdwg+0jpwwvqVieYpFKLIQrG7h7#(sA@MGaA+0m6q$d92H+qIFy^^? zkN&Zxc*7qZw7-xnYhMG*_8GAP|EmN7${7$K%=7_m8k^1s#_k&T&ww#O$u zgj`ZhD(cI>7k+84;H}B^uWI%%5V5VzHcNu#+XI*wGr)-#1`nEP4wblET=U=-!w|7H zRf8K^!VNi1c=YjJx_EJIx~gAxz4|qo>J4SZ9mGcrsMO~DhMW&}AOq`cng(9r zPC;Scs8o#dw%8vl+4tH;T<)Ll9eAXG@5*}E5#OAvOzpSh@@bF=0 z-Ix15RaI3TgKn`meBTh$%r~+*<{oT6FW-GCVR|efUsT`(t8(sfnS_dp`g5+6hn>IY z@x`3kW+76)2wg%8Sktl!{z4_POltbek=*|tDf!=@Sr?AHrxvVcTEu(az+l>tg^^|! zgO6c2-t=RxP`-8LKR?a=QayJwa9)Hk#J3hz<-3cgx_gX{sur0jRJ|LI`^E|~9N;b+ z80DtST10&~#ga_hm0E&Xfq~uksf>rM)XQkQB4fR|XHVt(f8mY(e|9vM@MzLJH~FcA zVjypDTy6Aj7l@G%zhB=x-3w1ud7;*!^vmr#uF65+9!{}~>bq~wbRExdq^>-N2_(6A z#V;h+-HkPC9JaYLD90eEK_tyly)9<@>kCkMC5x`T`NMTid1|p9_uCCipFVmrmyC08_knB1)dQv-MBZ( z0R2SVdfS>Z#K~^z*qYCaU0mI;n)2b(dPy6v%%q{n+#_5LAC5#9Y16z6RbeUt@n$l- zg-WXdyf^vtqmz}H>C=rWSAA^OP6PD(G4ge2oONdCnsdonFL}o)ip!MEQ38*7c4C&< zeI%cGF{{Ywa-IRtGbPFoalg(NNTjW8cI@XK6j^Tzy^fDsfvl_SyF5)WcQ1H%|OsOeyFDv1VH2yd7TZZTrr$T(?{rV*Ia8~38 z-MbmHayGxKAO{A16wc@BQ{6l4uwJ|9*rJZPPi)ji?v$Gaj)(XdWIRv4SYsTecA_fb zM`qRCiU-|kDE|x;E**P-y!63a@615mk0J^oIO;r+c|2DrT~d3cxUh>b++yE<5gv5T zDVC@rDC^`=+Y1mn*CzXqt(zV=k+h66G zdh7YoLi-tt4{kzVW_`}Yg%yULRikAt&MLaC*29lYU?=^+sL{HCv54pmNm_@6@IbjM z@!`Cf&HD5uc-n}yfv?jne=Hf?iP&-5t`YyX1{=kMQ6nlhF0MgMM7w4qA#X*$Qk#RP zz(Y{LWj#X=2*--&N8M(+Q*@L61hqCh(v_YT!{5kp zS?#FSV7lEg>?%mM*idmYV^VUMiOK5@OdfFg{lI9W#z}cmXeC#^mH~bl1r}hbWMyJd zF*-e^xp2PQ!S?yi+3czeq$F9b_?VCL&}v4SPK?k7bmQ?{74yT* z@wF#UQ7bEIcH{(H7U*{SlQO*)y6RM`{8%-}qx-$^_D}5VBc{hKnNdLeW zHLCqq(8kNq<|7vr0%3YNp=j~(i8qVA@@na(i43LL^bIA9LhkQOU{y0s+!y@ZZAWt9 zjXF4P*-p#DGdjnUS3X+0G@LFL(LFGTaqlh z@aiOPA_18JYgZV3Ftx=w~0;pbQt13Er>^6hy&^&MG_JetKJY zkya>O`HN9dY071{)+fBNNn8yXs9ZH6$(i(u8l)ld`75T!W&u?ac?yThn;bL}1O`8g zD;erYLW`tn!d6C1Evp~JbLwSeon^;$(0H32h+oTBTt}?^IB0#@tff3$aF#FCOnrMa zUeUud15EbX1U9(AOp?^yvowchsvtpg!sp2tcNhGL(o3SSUn9nx;3ma|H1#|Od{%L@ z(&GFD^3cqXj*S1bNqu5gLKMWwO?+!v!Lt5tc+Dx_4ute0C8*>vM!`acK-=cfi|g3h z4Hf7HK57H6>(6Vry8egRu8DE04DU~-p=Zisk6;*Nrq4BC~~GJ-L!w`#1(|<9SpY26Nnz* z)j;}9*Qpj zdW-Fpa)%FydET#B4%nomm$2&+gNywmWq#_Qfdm;7n&DyJ764p5-Z7$SQs(?~VSwQ-eFxyiN1_*yYXVYQ*%c)=+n4=CRW| z!Yzvl3H>4i5|urKNcgE*uWZ^#3qmEqMsW#w$|QR`5+1DIr->W%%TMS^%8CzjF-o%y z@bZ?C)h1+V^ohcT+8j)s)Qqxg`V)Jj9jDYpVCpY&`kZ9#=1xlur8VbkNLFB%kloBI z@#Kp85z@pFU9{%07R6CCY$ZwY-~u+QKzwseG{rUOeolI$TBDs6+@$=0toYfRjY2kr zgkls=<;?Z?3T|q&IQO=&`sr(TkxtQ4(+}Uy zjbcnH@sSIxmJD_$Wbbp@<6mwR5F5f^yo}p@+ZU{$(1JXg`b~v~OBd=W6mF`6R8x?G zM$AOMLud5?@@B6V27O-peNFH{Aa{`4;~)!Jjaaqy+iC^X2cr05R|9#1GmN(^_4sz4 zAIr0^cf8~|=j@`_cJJ2>MWlLIw+@bioQ~{x#SG%@NAjVKWu{IQGs5aoU)fFj3o(#n zxxiP)Hs503$&Z@#a}Ux9H=~J8eWCs*WnG$rHi$#If{dv~XQ$f)%fmVVrS37O;I3nV z8}<;D+}p$)ifukw*cxY~Zl9LOkZ}7kpan{Y1Ut zMzy>KgXz*wh5;{BGcH2fOPr68u{L}~8Mx5WdzalR^?EIt?jqOrEz%&E@Lrtf#sIlA zeDfRyZ)PPWMJ9rW^^%slPPiI0oaGXB&|!?v+ZUM#Dm*#;qZbb*Z;$AU3+jm;78bYpT1m`H>gqXcn5@sM1>{N1KT9C}7v-0sY#Zyz%xpsiZ!+My${?a8EJnhr$rR3BYVYM>Od zA92{z)6U$$Lj62qWZhW^=rEQC`_gX2Ox^ak)#2O#AD=R7L`4QHn?IPZfIm&j(yycN zce9ukrFwI1U{jn@)~MU{o5jy;nu8H*t4{(l{W5(UH>w__p^jGuQ%okuTgiT!IdYAD zbwjB_v%kV>^y9UjXd12gz}UDfpVWVmmMs(eVqgSeBYJ;Tlr~l}vyz#8`hh~)N4@HX zJXQiXxn!0AUqBem9e$;z&3=?z=ySxl>sZGZH7*et^SJh~(f!%25)q%-LNQ_*&XvoF ztyD*Nc5`-e<6a_)&kpkIV-1NGQw($b>7zkmSx3%-)r`-{g{l$lJA7ZA5_3`~6iv_h zupNpw>9;xzqPBM@tLxv-4#j4@MGRww6^#g%Hzmhzmo;=D^x-L46WNR+-Da(4-Y_Y? zO|9A^Lal{+Yq6Up%iP#J%(-)xY4uMR51Q7@lo5UszV(yI)z62UP^0D{q{Fs4nXWa} z`Fjr=t-%8_uUl%-;NopqLuNk;g~=OVRn&EfFt#$%tC69?5k%ilT)+nP zyM)8~EjK>|ZI6)fmYznmy{)|+=hO-QKyfjD)@W_94^ce|dMV@oqkE9zTW~s|}ILefj&3s0QZK{$m&f z&Rjw)VbZ%zJmS0_TN-OeVgq@G5i51u8?N}$)mLeX7H0NxlC>{A{CYX;*va|KKckkYnanI=l4RV!yygGk7 z;k^fZ%jH>jQ)}7~Q=m{YCF*d`-~s#A*25&?{SZGj#ar|2 zjVR%aXGMXc5!PYBhBr&4d&l0IFawB2dKP8I_7cY=D=43GC6xMY;oEv83)^-wK1V8M z5-9uPtC#y==gXjZAMpG^1kiJeVxm{^Y(eyw*4nAxCDs_NdvB zAW8b|ZzRDulzfK73oU+qHYs)Z_cv{K)k=)_BPjWo;q_wDf5-PoN(bu}^JA zE6+G+KAdTG6$7t9nZ$AJWIJkv+d%T(+#`ZqH!UuX?cN z%XWG8TJ_nui5;!lOKs^FxQ8h(5DBO+(emELIg1|fM>E{pa1*1xM#0;s9AZ-SMR#Wo zX?fCFgD21E5d5rzdg!yGa=!D#G$WcCvMCUZR=1g7tT9kYo`%sY?J8F`Y=Tn)e!TBM zXpfDra(;#vosX;ckk!x};%-v80MVss%SSy&E{z1;6MW?CaY0Q#2tc{@7|{^TXUwp@HwD zL?->G$SsY`Nu9ohj;qu&baNFhd2RT0K!F-eC{sI{>T+X$#ZOXhya%?FjqqLyoc3Z; zebUcD+Ymw9TlssH3h&*>^lp8SYh2r?g@4Xj@7+AsK<>G$2Mr1WvnLObKgS4qXHQij zRHbS^;HEs=W_iS%BD}2jSYQ>&E8TN50md>KTW>YWw=b&enl&Ojm7=uGA0~dSCc^2` zq+t3FjWdPEf_&}xYlg0?m|H05i|53j>z-QZ*KN$eT-Oz4FWDOD$t;s}vXRags#Rq; z=N~?|DM5Sfx)N*bSlEmW7!wj!PRWds-%rL?L=KrgjBi&&Zi#kw8$O7-?{d80BuO>Q zu^Bf>>Irua*oF;1PB7aqVP37^%$jQP7&N1+yDI`l>OFvHHQUyRZzUl{tqMNPo|N5= zk5t5ECA!W!#!%ctFGlR+=oO-GZS>`~=)2rUHR+ZKF7Tp>-rnRfQbvMoi;VfA?WcHb z6qi$C&9R-#j&UuY+OJz)MjS)BDq#F8S!-x8V5^OEZgpr2w zaho&L*wD^5GQPif@w#!kf#JwvK%30H2s!9%-+gJrbPc_os4o|d@Hcx4Q}6Zr-UTM? z^67=f5M!gE^m>V^O7vc9@u6~(b@sGW^M@nvtG@eH`@&M4aIXHa~{GYzwW~t$;P#KH^c>3yBMIp?Wzm3S4?=u>dOCt+)^I__&C+{d1D)h_6CzUjfPD21df0hbV;_KzOn zjtRwE8HBvrKScoz}b?PhPE+4oOnCRzI~&x3NlBFl>O@3c9K? zp7L;XGZzl(-FoRn;A9SnS<;VDkndjjVHVnvE;9<$iT6im^Wb89!x>|>Vem1XG+Yf|7jnjAT zc~R+W)^Gxkj&$Yy<#r6mcEKIBDh=&}CQXH@v7Q?@kNNco_)G3rHTia$^sx$ zf~~eyVpJpz37d2itz{2B<+-?vqy3w12b_8ZgK?7&7M%hUH24@t``%E}ELT4!lqHky z>UhSdND{nPe`=a^#x|$;l0Ruk7h`Xh99z|ly)pW^%*>Mw`*00(}}r6QFm7feS-IuZy>KuWOZ_&5^#huFJ#mepV}NCXAGWPKNp6qC{l0 zpGmz#HH?$izRmags@?GV&Gl*6FnittFGzY_g_C~pbJr<>+Y3ks3u)8_D+bs7?1R1- zA>7X{J``N?a=6l@d~`^hdH2P5dRp;8rCdk#USY*MqH?gV4^m~7JVoLcRHj75y$DFl zDk<~helPkL!)OVseX>IjYFilHu2Lp1lZy#loUvgIU;3MKXpJ>`5H6+Q_xe%~&t9YS zs{4>yT9YpqC+?-A3_ldJdrL*Lix0#U6ZEw!(uP3aeG8j6#z$(`#1=Yhw>p z9tLJM*kYEqe6tbO3{HuKc#a`~0dql7;hAXl0CGpk4PhbI>Mhg!Ch^TJ$^;_Ad#PPY zRxAfqv;1V4Tg|oZZrCvs_I7qlH2_G-Xe;cb#y-6-m9$ghi_8ygis~oC1Y!c1?1^a# ztM%rXUj|D`)W_qNu%`)eJ27%50V&4A=F%p;!NoPRua|3fHs2M+#_cRv&2~Xzsm(E! zg{^eS%PM#743d}Eufma34LP!d$n=@w zs^agTHW!ijiURM?6hLp{SE-d!dAQp?ZtHIaP?4=B1bF5v0n2u5-3Fu=XqT&mU z&58A@lo3jzVQLnxUDePsJG~}&@+ZQHpyk)cE2AKDRodZv7fj$o3Yb8`QN8W5L&D*h{l-MPX3Lr3y91XsJbR~N| zC(Zn04P5!H=;AL6eJny$*pjNxX`hYJC?z`W1^pQz*eB@H!T_EQY7w;oA#Wf-|iN7MON-(a423nU_ z;5tna5O$f~r)Tx0{O{ggS_3iZF3=?3)T@KU`_San6hNUul0X!Pd=5e9brsf z@E4i??C{=cA9X@98iqf{m}&+gV&>>!_GrTgo4J+qpw^(nNo{3Or;c*4^R$}!FdB1b zz8~`OK^M-w9+jVg8sIIl3sRAF>^V;zxB!{pTAVn;>NIGZDI%Mr=?pp2?F}0~8aZO7 zDe8pN4RKw_RXX4sCn*cj26HLDR^ibj1=nOo@((Jy0Y|p~;eJXf``XAyfg}sdNDCj5 zdWfCMQpE`eS5~pYRn(RB_eR15j(oov(3}H(Rx#HMu-= zS_GWlHE~S;;co?Jz2ev1H3!*UdJq=n4(_ZAvHj$?`+W~AT4_xxnu89xiHz`6ymDmAIneh+G6v=Z8P6Z{XqPnj%GVxt}F(&wu!a7m#h z@~(ca6RxozbHhA*f3bd*rWpBX5~6KHDX6fl@WyOyZN*&)jB(wXLX+>d?*1|P@AFDF zv){{Iaa({#1gy>Y4|??bkh6#er?U^meuj-bCKkYJkF~WZ7kf$TD*Df#I>@!p;m=6EG-Ae>xpI3NPH;=*}LgXBm<`kHfxq5vDu2caLoi zJFRk@S-!M?|9)KrKeyawu*_@t)omD)^ebK?9tbz zD;shvIM9dZe*2ELz@w#fYYh#Ir*v4hYxq{e9jW-(G-C?Mapv#;Tua%V*P)-9{zJ8w z-MoW4n|_Lv0b;Gspt`MmY8Goy?P|N*UFMzf_huB#zA^4p67M>lt_UBhb=}x)6^0)datVPthIyA|+2O z0lg#5UI~ImpB~)}nRdG7j;-SyWC8KhaU^DeFjK(Ly zQM`Y@FP{=kK!aK55?>m9Q~*pZagL1QE8)s-Dufb$SXo{!?6mu)vyZ>AV#xUiaAS3+ zcOGoFbyu1ynvJ=O{rs-6?RI{jI8xR`DmhF{P|(IWzZnTec$Fl4>+5srn>?1BBMJ47 zYsSR9TZn3zy``C%S2<2yDS837r+PNMc!8Vi=W2P--j|G#jIWa|9P(!}nN!fxcT!UE zu&+8cNLlQ9>gex(TlUq=Q|^80%A~sqDuY`BrlmFMaDvtF@z1iZZ!bra>};?vZNWgv z<*{h;NOg*D5XJ^^>h$U2mfy|Vls&8320nbaE@i1~5W(4P>y%{o+ zRiVgS;LbeGu2%H!-BmcV(>z&e$E|GGUwZe+XM%^p`qd?CTaSbGm9G6xpFU<&w~iQJ zzKi__2PY?NWPn&3Xt8^B_U{ev(mN%&Q(Y`80jGrkiUSPq+1!A&1&6q${z)-^RU$^uQ(V6D9x zxF{dRG+;~3Bc{)`wP^KYXP15Ib|v&!=?@vNh94k3-VVWHbmg0(0&n>HtIeiAX~eB| z>*4y0(+e)r(s1n@moA!>L!mzdGv|aaFCs9UQ*)8>>G=dAhmJ?}EFHXe z3Un8X6)l&HlsbPNo7J@X)?ry$L({Jx``fFb*yJog*_T*V}sm;EC7F4?Y5*A)Vb2j*TpwtYBj zQEgo!anHQOGEe(to`ZMCZ7}x*_Wr|-Dbn!B8`$nul>T{@1NnXaH{H&sX3KMSRnHA{ zbR5<5e{h$tcvn@&o_{YrE%$HY#O!M1?tk0|{?)w#ix}VBtVzMU8fD*9oocSwaw-Qs z^QJzV?s^%@-v{{u9ddBDR4*OxQ$uLvB-WtwFe*@b8uB}%c+STwfOLbh9|L>qZzpPJy^0(=c55G@*T5<+!nbec5 z>Af}+v$Dw5r~viauFyhv2UWKX_?gL=ceLE@41zy#u24o<)Ec!~#TKOj4*FkwlDtHj z|BW%fGE)CS5wx^;&Z~Eh)nNXQs_$1Y3YWU|Lay< zPFeYSY{m=Y`!^&p4n5cJ@$==+LxUjm1NM2JoUZNwTJfV4y^^eR@o{hJcV`VfJo3j5 z$dVo2B|{QCdvv`blrusDx}b%Bs`32q?00;I8*<`bT}I^;^UGI+9SMv@82GaFwSMp* zG*9_b=`pii*JH`#yRuaOx0XA!-x4Ir% zh#VP2tT#of(p>33=(lv@w@!G1f6At3Y^2uDV59#v-jKQ|Z`eAF07H^mmLP3iU3e%o zNE-q{$-*XYgHC!jpu@4x+kZ2>Qn2nA zzyJcH^lGEGs0bL4PC!CaQEAdkq+13UQ9weI8hQ&5=`}<}KtMoBKstzYNg#9r$vp`; zgLf2@!;T6MI zye2(4%7`u_$#t#wd!cmg`vpbj-b%LWa==ThQ&Mo=*FpQa?iNG4fhX3u_--&88`{Kc zg#TOMCjANA$gr?QNt67P=2k@;A9F|LJx3jlhZxHpIq4#E(rBBqu<)-tnOPPtz}882 zc%3c(*xm>9cd1LE$~-lMQ1I+M7>w?@Ig^DdalyOf!xwAWB}p(SIKWmI5D)+_xu3)$ zmXtL;O^Ay-tM9F+9~l#4bjLhsyx;GV(dqvwh4HvZ^_CoCqUN)R!3Dt}S)BLisf_J) zqHsVCui^zh^ZlDlNZyHodoIk(IqAR62ke_Unh8dF?aT$B-FTBf5m{b_RWuI0IL>d| z!O_xP;cPiS9FfX);dm}*wF~~dX53koG|L8IX=x({7JgyKh+#RyPM|*a2OqL_Vma|~ zrGJD1Pb-X#Ws4)de+&O>t^i2SmvO?M*)-@<^{l_}O<%plQFHw0^3qZlEwv1qQ__uT zGaJI5q%H0ThjtBd+CNJkYf}wjpII?>%yzbffT8a1|6N|WU`l7VAc~s$mLtNvR<|Zp z8vl9VfZeC#+BCWcGcd@Lj;T!nC-d=R!gN4R+U3*=5xs-2jRDIbt`aK0RO|w|Y9nLs z)2>h57~;N-_4V%8M|OT^caoe!Os|BDUGt1vv+Xh1pKOE%_y2BZ&L$bz>zW)V=9w3~ z2rrD=-uwqxJn&+V3_x#S@;%=`bE#8C0iNB~N6T^VL5zPaT!ML^T590EioL0B&AFB; zx?vZ$AO2DmbLR z!#X6w#B9$jfW+(AG~_BW(y_P+W{{5jxr}a1y3b&DtcQD;yv&)Gd#$BO!X5hkRM?%P z(yfTO=R~=A`HPB0%lr2qBCYs)#^?-UU()UbF9imZZ(Mk@rFYaD+A;x5S$eQmWnSJ9 zqpzdmlwmo5+q!(=Zml4gxx7ivLJISU^xzuESF>b5x)Dfw2oTG0Zi2{|Ms3k!>S2n0s- zUnr3BFVgk(AF^0%*X&w{JOi%F&SqltV*)xsJ5avQVI;L@!lemR;wD^#LHOzzD(G#{ zoa2MuygAm~CaXJ7u0r50d3e`K`7EcHipNO#)|i7?Q9~G_@tu24POgA)!|S+%JBl*U zl4lk*LAKum(qOv{NYC_{cI^lh3LHfOlea3YEIxpTl-yoCvO@5+o8Mlb-JJ&_mv<&p zIe9-WP7MlFOl4Bq;!c!xg(FY=NoexTu$TPfwrc*dwVxL{Kj7ZEc4!smMXmMq2L3xi zwo{CU3umW0c-SQ;FJ+|rq#0SYbqsD`Y_5HAG8WYYQ-FMVp?hbr9`x~M56HW?gCM;) zXoY@<%9%PQZ39Sg47UdPoF=+A?}O`N1hYKhCK#kYz36g|41bac;~ z9ka*^_-jets8Mmd>he47NjA16yJKQyvXS2T<7s3!C_;p={X)tN%G2pqCQf|-T7ux>gr_* zg+gM=);Sszt^Qxw&#HXg&snEJP#yhykLLqhzenZR4Djg&5o=`=FXna6=UT@2H{2`7 z<&x*Z2d3E}JGYEQGdV<8o7n)yM(q0mDX&f!q2j0Ip_Q1B1y11sTK6|Mp;H=bpcJd1 zv^!Gz?*m#gJh?zje0hq=ZamXJH~E$M%>DcKf8&R+Wr395wj(v5K|xN7wi}U=RZ}L$ z%6YRRL@r62PJ_n;tTsp{)rXemCG#i#mW13pL#In znm?dqbp3jTxwm_*D9h8@!SK8TPk{p(FuIGA;xRuippf6#sQF=RSvgKl(OiYO-~*Oj z#7L|w{jk1-+}%7ESJ%=TH&aPV_AxsKXvsZ0bY!0o{6G-Q8w@aS==E^5GKiOIZkBV_ zbWOh2@-lj`_TIklR4q*a`TV zW?`3et1L#+w#dhg1(rXK^R(dnZ?6judAHwyc5a3#eP~8RN8x(pa`X8$6Y|RJ`)59k zJqBi&(dW;L`?d(vo++8KD_&M$U$e2W6hN+>MDwVf&#mbp7CZz36;5KNsSyU(OJ)6J zB`8n+O=-g|0*stvrfwNvsjsX1x0+N}w%z^fA<~-T)U$A4IOwpxV-Q)oo5zfr|3M37BkEDeBvl9oLg$He&4u-TBz#p=@+-PC5lW>PQFpENPYw! zlEL@XC>udEu_OLF6I>IH1Tpw;*}6Oy*dHNg)Z)93&3!!!;A*8}{#a!)BLMGO5^;@4 z)dH~goVV1m7YejnBb!=Z@@lRxnGv5P!uvKCfPi)Pi$hUJQiyq?teWc-F7sSr-JL=S zFsXMf+@LH`7i4#TB?Z4wn&?5y2#j&2-Sb~7D__sJC3U>12g|GDq@n@|(T^o_Z1efW z(HLTyYHiW=xa{O#aXvL^A5!==<8r|zA6O@gjNC;_pz3sg1+pHiY>Ylg+IXQkBlA5e zbwM;c>eNrS=0~WBr{D0jG#clp*mfvf0q5aqD;x0e(&Mz3-j@taqUjfdj7Pnmagr&S z8DE(g!C^_bvd(I2`;(!d$B`hRl_^|Gj-S4HHK43>S zx8?YuPsg<6AdyHCQAEiyqol+jgd;{}v2AvO#=iS7XP&o4XGz)_MYEQ&(z*A=fLj$M z2Dt$u?)g1BJ@^s3ks50{{$5CzomDtqMYVfN|7hn zI@BZhwG2T@5C!TuPbnh`fP3&jfiT`tRT|%&9S){(h*A6Vz(8;)5^Q*U`jZHJnWj9a zoR1E$2-H(;$dIq1ZI1_=yTj&3R}W4V`)@w2{kR$Xvw5(-+cFg&nU&OKIc{m7b0g+I zN%0mXm6UfRrP!|wZz#0~m`x?Rb(mzG;j`X&~Rcc@veWzkT8Q z{}-OOQ#r{7eya5M`g!u86{u*Lqqbc(@I^h(b&T`H+&@F^nEVr{R)T`rmBTNp4eX2@ zKcM&_n0?fJP(3i8YI*yiZ`9;Kxg!P~XxEvqynH!N&TIL1)euh|A)%3Je5@<5;I?Bw zE^o@BaeTO;d-z3d4Xs+JIdA1kvz{kfYH<1oVI*32;h$Den(qZtre;lc1 z*sNL)P!eQ%33peCSL+c)R)}%oIG5qYix0TCr7s~*j1nGI~67}%)OJN8@o z653!_7P%JXDh9Po7(k)WjWuRep&lHb+Yr){djD_rl9s3a3>#!}3ncmSIa9bW{!LUB zxUNN6?B0eWq0)C}1fAZEIaPSGO=O=s)5-x!E8Q_1`HsxS!=7Mi>5t)$=J%R+ydc7f zE6`_UW@eokeJufvVrc#ffFZB+Qgm!?4t8qO`-%BL5FYab8bDRmUU$-(}aLMDfS!rf!4r>QB7!K)e zeP9k)@baY-c6HXrSd_)LlE;{Vg-;y`@&ZYF)f#@S`|z-^_~g2oAPIM`nqXnm6mLnx zy##AoT+H94O;?fH=+T$9XT%hT*83d@&uqI?jg18gY#1YL*m{ryG2gWf5n%;6 z)%)dTjZm6LWBaac$lt1Aw>>9p^^~i8zb)`&3;k8>4zqDutt>_v;+9eZDGv8D#%T#w zlp{8a_BSXfEO!|WD&=u2%?h;S4O6RN1jMqO7QUT|;cq*nq$Wm9LdKuE&b=}xPqt}K zid1hT4@v4^y{R$$iW?RDl*Mx^6ucN<3HzW(U<~mIh01M5O4+UdPA-|S;1;1&+JJdB z?u}uz+p{m5XE&7^Qb>JWX-}RC)?Sa$N>VVMSs&UabdbuU1qB351YEUkHwHA^5jbyN z=uq{@uOMyaP}K4afk#9MIS0^()FrhRFndZKNaF5MO4OD3Ve`1#fX&({I1_q!-i`d7&Fp@u1y8Sm>trDcX)$aNFGxoC=$a zZDEj&t7AXqOE|X0Bh$mA@_Q$>rM{0w_c+}CLo|9|va6CK7HpVL8qNcAM-GdD*2F(# z@`vZe>fkQqq94Aa;dxHHspE-^P4a6x2WCtyd>_y;rkb`_!-i8?Xm4rfk8tW=^3sKA zE2lD>A+4n;3Qo9@RXhp3!ZFrfnpm z|Im~vf0e>J|>0ZU%aEkJ8oUeL&oxZ)c*8W76xHxjA%-a-bc4 zw!AqkAShrC!K9FlFv{&U=8a+HZ)`EV!?!G+zQwaM~GL74zFHBhzwt6M_Jt{<_ z`K!iPm5}LsYsZ`IbhtXZ>yphb9##v4I3>hz2B>S#g7%V@uzYy&Wm#F{%{8V&%TJSV z-Cf!ZGftbQI2FCpo5_hnVva!!`INa9$E!fS|IV=U;rEukt_L-3HY0-bSHUV}@BtPpp_`U3q z*4+dywr0CbNzPr^Xk=bf3^EBm6^pe>6O=<=I#>}Zf(Us3U~tge9XG&4Tle^b~)`a)c*xH@!I;f5bwnJm*$f|HOS+x zJqq!BP+LyCAuVSckeYX!Ubxhn2+?3;i>bXsF9Cfx`v;&+AI$nD*^1j`9&3>MM)8oo z>{=WFvp$t+v&F57uBiBU3{6jQDvs|~o@lrd7!V*LKXs)P<4*K@{AJb0etlXQ+L;;F zi>40k$8T>Ysc)>#sark|heYWjSFy4Mz@S~7Co+9OOZbh{Vr@ril}lX%{+pm-ATK}z#aFIhW%GsUgI4`vHgN3S zfTX9WypPUIBESB65s=$op+^?hjA~mAZEAKTBz)V1n;hujf+8a~ha3QSUhV5M78Vxo zp=^%`omMFFN%%`a76ymIy&=5xSNqp)F_@7Ht1`b=lY#L8#-O}YJ?i6+A3cgnjDXF9 zVud~*AK%1?kxzrVjZF@_wD{9ce`~Q3J!5<3Gm&_P`rt^tlTD5RgV8KJKc!)6S#|Hg zHT)tY?bWHEx-r5XQx?hFY>r!Bfuj(v>|PZc7FxD6=*OHzarkmi z0YSZErO;QQ% z5;i{jPHwHgUwbd@T`a7n5hP9$hTTF)2H=+s1qFrJNz>NvVvVD@BHF0fC90|f`A^rg zlkFs|23E@Ge95M=EbMLTP`0PI*HtwpC#}lqTX%}suxrCgPyy*k;`|62U6A14Nn+K; z8Vn#xj?PI&G^I;PV7dY)-GLcxRrDE?9J72!L6)b-;5|icW)a+n)!Lfoo_(*LH*;(=e852SzXFw{O=EpkqEPZ5pyhB31i9?dWOq#I=t9V>piDJc*#(n#kZ$YnIGN7lsXVp8D zX}9FnyQ%3P+C-FmOYi!OJ>6{U#OZ1~h7n9Vv#FlrVw;Uwe!|dP2E5I7VCg1)(&D43 z5FIWYmO*R5zI|uqWV{{}=jC0~-Wng@Odj`zb}r#483p7|F$XFBf(DrU0dx5YuPutL zVo|1Xyv2R33mB}{lcWm6eDeZF@d~pDeqTNEDi+O*=F*W%)PqG?XgJ_B^s`J;ukoX~ zA(ZU0#0ERsEA9L8$48fTNT$4IlLQT;wk&JU3}J~!6$%*eGiZxSs0K&8n?z4OilST$ zFLMy8DCqBh0On$*dyZ5<8638w4KY5~H3+8Z1nZdSA)FqHIw_kw%EI z-*@P`pe^+COn0^SNXH#imFe1`JBqPnzBimF=-@WlKv?p_mokE%euRLwuFRkiw7=M@ ze0Hwkm?-NJd`@PS;rzTkoN}Y9k2*oWJF4K*{3SV)JD9Q@9p?Xg5UDm;!@$4 zTjRmrFj0+KtYO@OJ{ih5nxg>AOip)F4)?OF@rb#Q4$Zg8XjYJOyBSXV&lI%!C1}u@ z<=N}8L*V1+iZV*($x6Evmg?U2zDOp~C>h;DIInuNO`g`6+*s^Js$5*(7>INQ1D4lp zrGJdK`(bfA z7AuQBriShxz)Ct1R^QDE6-s0j<`gZhPI!d0kKTOqld+}99nIwdGp$Q};gP4uem@VZ z2ar89Jap*R>|wRY3W@AzBeC3)-g4ei3~SltRhOf2%>tsf!u+S}Ha{BEUgrvU zBK#CU>F1#++|b+D@;o0w9eKkRf<+2yYx&IIy`T^**^33-#)de^;f#iDn`Vlx-~ z%zfjI(R8r3MgBa-k+a86-2PtYd$wX;=#%@VGuKorPr44%gr)~wyAU!x`hLJx$!)xs zsy^PBnB|RI!ake|<(lvL;%O1VJ4AhjwXGp5wZ<3gaB>poQoyk1^@Z0U1T<)BPlwir z#6b`#sU6upkTSz3SA4F|(NnVo->(RBzk#fLz)@^ogljlPo`YY@z{;xeWzR#u*E-KJ zk2viWlxf32gVMsnud&U4KY&F}kJ*9KH(+T{?z{W>P9 zNz^X}#QLaN>{}XiJItU`rAQ-*)mQ;yb8ePc-n*$pc@_KmS}xf9j4tu>5k7I}*%PI* zW~95JmKmoK5bwbf4z=p@d<=kG(U0O0+%*HTI2AGa{-pnt@cR$n7`_#Z1^>_nrYeY$ zhnbd_tfW!PTfYTDH>UcuZMC!uyi4!eB#aE_ z*VZbvQqUXw+qF@Ytl1gpbfpCa&1AbzqtL0!+@{-Rpw z$<7+b!Sn;=ei~>ar93`_xE`WJ11#X2c7FmpcD-|{YGTs!NKhTTaP|jDiEAHfWEo0* zy`Pdcw))6?boQ?L0RZDM+p;TP#K$6qSKrQy-@Iy>i^dD&@FHD&qRmCnyO0k562l84 zJ`=L|Qdk-Hrlu*6H)qQU95b_tM+?1=Xwj(0XPN)_04=+nFfJ9!J$h0-pgrr#)@KI3 zmf0xBXP8k^j)A&~xvw#@yNax-nWUgH=9I+?RM(+g42SefQ$d5QxLJ5)Vt3^jxPzKg0#~h;W6?#9B!+N|gG)n_6lza=v^apgn2ehFfEL_RjX;RvTcji$!c}cMXOm2A5kktjLorBu$vwedv*hQu< zty;m7B15l{axn{vx5}2(mX4&`GGkT*&kx5Mvsh&uCf<$`Ax*!kZ_Cp~HZ*Ro5t;b! z-+}dmCx(H?ayI2;HKesICB)v*HIM7m<$;DONpnH`<6Qu!pGtbIJ%-)(cE?98X?qbq z<;2*ML!)ozOUGWbRR*({tv5`bI5M?ETm^gBm;xW4lja-5^J$A?hDBzh&bUne%L*Zra!)4pqbFW{ zGjZdl!Q^%I`Z0$LQbtkkV_NLAwu=EV#Y-HmU(g-Pt;A(d)D<2SvAdU%_rwnn|7_k) zoPXm}ugIe>Abu}Ch1A~;AV+G2>1Wmsf*7MtDDCD5&2}^`-4u73EG()>(++pK5}SL= z*l&t3@}CM&dPdp?(YQ8(OW=w5CuoT{y<6%RZE(ttCBO4Chj`3bXYjQU-;aCh%3-d` zq?$s*)5;NR;T0e(@`)dvexY2BP#PA{mdU)`_lvP#kk(Q{i%3-#6)wp3;xoY zaVv9JQsDQKk9>rUWyIaK7kp6lx8v$o=bIXwXjrr1x71`i{2KZ z%Gi6#Z^1?1c(uk)(9I|OnIWU&kf_@M6XhZ0M65rn!1++6q?n*b zv*9Ov_--MDvJ46_M$BZdd2vhYl;oPGgtYQCXG@Ex{bp|(84gSgI5w4!6jdX`V38iH zs|807fXL$!Usqn9el7^f?kZ>lE`+=Qe@AELEBmGZoz~;+%t5TxrnM-!R}w1clsK&i zFyJN~gG)uJ+q9t1XdvMTTuxI&mf%SO?&a{_iGeKHmmXrQn090T(E_efpN%NiLw>_$ zU;rGVWI9&_lIFj$ap7^aW|lr^Pu(9@-1p^G`-YEsef_rX)|VLpNkrQ}@;$AwpB7&v zd3uar)F%hgS$^%q#*v}eUkDh9tKJz#xHvh(V^5ylr4Cmje$89&5{6m{?AQ0xpT%R8 zFKzdc8<+b+BS8@8HlM-kNBD-040B|LMt%M=kIkqxz=WQ0-CFgiqik!lhIYKpy){(S z-uYY7mmKQDV;^y?8@*nr;Ufn)vDh%;NFr(>yZE&oLRLAB- zS+$sjVMZ(wvIp>-?}eWqmGPoR4?2r|4%{9PYgCsuW#9zhd}{?>Bx*JCgTLJLsl>Wt zjhY4LTAk~hyEGL)tIXsPW@d`36HaYs7nyo%)KtlLWC^iZnY-4c-`aK+%r-VuFAmL^ z7tcyYJjHrt=MFBC%Q*_p4ZZZ&AjlY2ja^6;TgmW}Kiji7FmSB*vcpH*keir6d26K-Y9kpoKX0}qK)8ts~hT_MJJ$qw~ftZSMpI>SUwsJuz_XesOjZ?>_0 zWc^=?>BSXe6-|-l)K|mFcS*{alRg3ooRL1pJ zZ^1SnqolxlQfQ6>YE=QX(+ITZ=C+)vzHH>PX>0HAJNL4ux}*?#!RkGdm+zBv#s#(s z7oW5Az2Km_%Rhr$9dF7UTZ1@Z*on2;OOv7oYK>$s>sED6B_F5jbAx+HNF2F;g=HF` zvN0lNV6AC+()7#Y~PWg*WJ(mBtMfl+IDKnr?Rzc8>or?sDliN#WYpp2MC~ zc1`mLyVibuoFdUq&g07qN^h$C7xA5QMHhEocJyA_tp`i~TlLqrH|txy=JL(tJo2<5 zl%}-J_jr`Pr{_lU9dzzi6>*rn@^(CaIbt%)4G*KX1_skn)*~f|o|;)LJzEPR1yIy1 z78e?n{>aE~Hd8D81Csp5Vr=`W?u=Hd^+?#wR|S3a0SX)Xt1Nx zX2W{q)wkMS1-(TEx?>zA_k^SLem!d_i7&NvbnGG>ioa)1KaR9-ie(2NWlRc(4kR!b zb|Tx2oI71%u&CMj(t~2;;w)76pe#elWXei#qovnmUr0Y`w&g(Y+|tHp`NG$&Q+#2K zw`Y*)7wuoI%c0rCuS zgpAG90^+ZQV>mC&a8t6tuZEFJ?yPm+?ejUuTIWhqJQKlT%z}v)LtCRcj^O%RkE2WT zTzfw+t=#$JNx!yO?%I&L+)fL6>%6zNF(}GL-BY?FUu*%lF2b{ z7E=#Ea@-m}skWl3bvmpK=D1DdryVi$jj0MqGwll}srLK+X0J^&$iYF<3w>vuQ|094 z14|i~mo|LTW{E=yV&A!b&%K4&J$i2@wX%XvGV6C&aXR{Z?^gY+)%YK*F!^6b>Dz#Q k<)iFJcV>vtzjF$qG97t`L{_V^(nVINiq@5a%QqhV8#9ayJ&gjHo^h6|j5H(u#-ZIL_AQA-8%V>j` z=)>rv`@Hx5{`~{@K5LzIo>O)^XYIYtE+4hEUQv=Wkplogsivx|3jm;zn>KojR(Ih`2YaSMu^~0#7=y6e&9tV^DxX00IE#r{`e!R{prLK zuf=xf!4Ja!U9AGC`?gEa&)KUpoj89g1s@{qd=}1QfLRG3?!Ql-Z^aUB@F<*M<=^dm zF4BB%0`9uoQn38bgh?zQdhSxDlLtKJ)(%$#AZFj1u7iJlDaFLaucSye({U}w@GuX# zloUdVNLFr%LNlig3ex#=vxCLxRm0Ym49M+oKBBugq*dPRMd`JMoLDpBTq7g?HlQx7 z{jt=ti_FYDe)bX6g8Ae=%&W7XMxj0kZT|>z z+dECnF_Gbyj}|*GdhQ@HD}Oo$+Jn0!Dopo~Yb9Lo>t>VAtiih!|5+6B zdN1~r2}EzUC*)MBxHu$v?702G$LBOPcyQ|2rF8YUl29TXh|ydR66?4Jt^6V@ksizN ztE5Ei+HbE{OKCf|IyA+@w|tUuATiYuFQVwL!Zfc#mpuuyiF)3JBoO zO3Ae{^;z7?!CJ5n`l9=edl>8whY~#4*_7i@*#f_R?_36X{RdQ)N&Rz3G)HY}Ik38=GR5_Cz&J}GMw zta0!e7OE34yx-A>Yz()v-zX^8EHq7iVD|GzmL+$L?QQyE%3|7(T@77-Twe+oo5l3( z2rU=7|K#$Y_$Sa7Zwh{BB5e!iFh5v(dNEimZ7J@RUa$!2MD%Himtdo=$~=2b8d*K_ z9~gQ{ss&B1=WsB;#U?N$bK*N{&DlY{1IZ;M9ycfS6(Js88qrpSjXJZO8SX(}?D?dh zR>m!+m-VNdvD@IVC^#6ts^^b8m|)quW&zwxd_{uY51%9-lQT3)h?!-|&ooBnA&BBo znQj!qgHPM}+wgrST;bb##;)QsrkZdU%#3cU2aN67(8&KD6RKw>o3 z(qz-p`L}5|KeH^P#VG^(uePa7VG1KQe4y1-bY2^?zy-)?8w-gYeLQjR?2`}VHOh?T zK1}d^cGxwqOx|`&Ee8P~>xQo`ZZQD;$!H4AtSr5NGhr8Al%!>Uy&7MqN+O)M%35iE znZBX7Hf2bVB;4QzpWhm-?sR~2f4<}AX?3Zo^D#+7KPmYr1`X95;kq=Lz|Vx?y{i5~ zVuA4dRjI}4wG}bAk3;E0J;RN19~O1dB9Ar|kUeFE-It(bN!3F#9 zobnrwo)ZDtU3XX4%2t?s=GDs09Trs2`}bTq$e$yS5UoRB64qz`mN`v^9{`4$H>{m< zPP=lQ%=bw+>7V1N2BBe3Yg=vtoCEB!Dl<>dqPCLFCTt}+Q=?APg$}|)2LEK|4ug(9mIcyDv z@#q`RMc0RUhF*-$%Tdh_=6&ytGc!#|yz<%zkqEV-ychxx)f4D9$ja5$$;~CAp+ta} zny7%wM#45s;StZD{5^Tf(#LN(50%>!M%|631^H&CEP1Q&W>P?>dzOuwI&QkaMZ*_$ z7ZcbZ&^@yIu8i7fhfU>>D z`%Uy;V_HKDv0FQF@rMVVZ5N3%4vUYnt2ap#=15wOvPq=hz)trn-+YWQeE zKd4jJMbaQJ!KBlQYXVcsN1W@lsJ;pJKB;ksQrv%^EC|3wNOcFl4bH?jy))cx^|8wE6VGYD+TN#Qf3v$C;I7C;YoeU$m3s@e)f3^6A-%xBdlS0{R~A>8mzkZ zOW33?;#~oFgqT0cnn}*t6YjQG2KJikD%rL@_xX&%yLc%Fbyr^bTMwT}xtcDeI<~9> zb?k#=%dI8V95Q*%$;7%dTu9p#>mYX~(K+}8(btigF5mA~hI2kUQb7kA-s^yJ5pady zGgWwKFr770TJtmt%)5{8Gf@uJ*iCih*0&B0?VAeawOEq*;3p7HpZYYZ=SzltsYa|| z?>e0Co32iiWkVP5>m!Ctnt8}#<@~mebn902tuH{@>kWbpn?E04Q!E6V{KXXX5vF$? zAAi>mT5S-Qs`mLL7GgURwm_%ja7%&gHJvK*Ede;+bR=se*8c&62aZ-JyCP1f3W&nO z^9tX#&WdeAipjo^GEBEq0_}x<5r@M_l+f;l?1$IGWbS%^&yC-c(dahNK+^R;TNPjf4a5RB;NlLfa zC1feb0s07dT5j;7?kK_1j zA}s(?A_n&#cAWuzJERhTmvnF*5Y>KABI~(q>;N+j-JQqpr2FV59q!6L@**&nhl7@? z$Kp~<{c8Z?!+rXmQl2mt$)QAYutWQf?9k=|r)Y%c zWn&mKMmmir_q``MB)luNBgpD>G}Wm2B>8foh&*dS@rIK`HQ%ut)o^KR*X^E*4hnb! z%@aoUHX9|-Fool4WTvvJv$vIA1j?c0dY*w4Tzb`psrtX0EB-{TQzOsgXFk9dVe@jk z!qLpzjZKjQBA-yh2A zGgCgHr0VEHk7~c2wzc`2i@$gMeNPf%E}~yu$17);pEzr7b{bg{CU?D1pUH(VF;*yYK}#^>C{!nbkOylspw?ZXWe7;1RE&tq;MQ3!nF_d$kZLVhcv zeo#&}mHD=TAF~ud5`|Ma&6Brvx7U3EB^dCw#k-L z%d)aY#QjGLtnmHJZ>~c8uJJFZ>5XZ-85tMbD7DY-=bI+&RT z!mFIw+zFk)Qxsxzg$p64#;rWD3Ias*qz&4q0*+>{%fv%ugjZ#mMG)x)b+%-q8_?qH z7BPbrLyhsPAw$_r>S-}ezL#}SK8`kHh3a6t0MvIc8nOBFY%%}p@8vt%l@G4B7CJIG z46G_Sq=2n_9O~Ws3iudnBz6Kbz7yKsoE7B0QCvn}?2&#EYwAe5$Es<-+yUNy{#tKt zE3WlKGA31^PAevFr4`Ee7#4?X;7y`7luG-hvE%S&`#JX>RzE(^?e1aFq&stoGF@ew z(qDBw!O5?6vUQ;`;H@dh^S^zTTLEIM*}NHoP3gscnEY23GY~y8F2h01J><|NIGMmp zaqq|h@4;a|5lksnzfs`Cl)Y+!Ri>G->wbYlqi8=gFT9vxe5%rvp8a~yF6XN4RW02# zulx6BywVOx-^b(r0 zOLv+oCURJNf^mpBUrmMM%OnitMfmBB$3o9$huVFvvxF|(M{FYsZO^@J<_MJ-*Ydkiykb-G_*4V(I8}ew?0^9Z$NMT4Z*!v-bkIW z$cp(t_NDYtDz!3;GhzEi^=h&`UTB|9-8R;z zBS?C`?7F#w*09IPbV)k1Gv&sK&Yf4C#L8EYfmGF-8JDP~TuV}Guqj-!aUbo|aoG+R z+TJ{Kj~B@R6PF90bW_ndj=qKa_G?c+V)urn3J zCc6Y&G&);Y!(Gk`d&;|&9|6TY9&sD_kuD;_R9T6f; z+iyACPbQX(-2>zF@b@o@G-PGth2j+Y?{@k)UldV>(W+ar8MreIH+Lt7g(ijAqNS8S z$zBC+`CgBN$X)ryw1?}=Q}M#fxQVP?=+np0ZDbQAI~RQ?`Fn6p*&y~$^hVOBS-wI? zkek<_zlMOBZWEBEiP1y6x{oA)BPrSVt=@1ch3&JL4IgINV*JbsTYM4=+Q}+5iRHQ- zq+7HJ>Zn}DVMiZQzI1p_$~NV_m*uW;x*rpx{K;O3?)6|8by;_(L^^jW>qWXQ~AiE*x+$QJcS{f9E zC)%Z2uQu;gZ=nT_4lc2?6e8CL6Z2QQ*B){Q!&Is_o1HDf51XCP!(oS47Og8s8W(+s zcF^!rlW}jrDTG9CZ!ldxM+EMy|TQ zk9*xASN~#m%5omn%o|3-_J`*~@E?9_QyUZLv+Hd_tQF(+H3g$xA|1%j?r_^}Dl|#@ z`kF21x+APoaHm;fjWwzx#LeS+wW#T~g~HLy#~j$L3Nv#pAJ(7pM^}p*%TT9g!PSdP z`2(~CDZx6qpW^87U9`nTg-@c9yz$kKmMMgvT`3rS1}ks9m<=w)Q^mdnELuKfR++87 z{Wo0mpm3Qy#14_aNpXQ=`EHekJC%;^KFj_&cIpHD{B{joCr=~uY^(Tjz%hr=n~&et zBLu(9K6v5p0UNSYS>qt7H9v`p4Eu8}qUR%h0}Fd5XpKex%=+u(QHZG+gnscojNJRn z|7BK|fv;KX@?l`^KMZO)Bt0@-)gUixPM+-R)|BSc{XE$E7~?fEa3*-?x_!_5vRHq3 zzImW(s%LoTvhAlWjRK>Vd-ar&N9K?y#W3=>9v)YzI5w68TxNis!T zZa)8g{BxewHH@d5Bnx%X1Fw~$7rkr5zWjC9;$Y@#`R-Sj4y#a%US;!?KHhW4!YWbr zUm>JpIPXEY*ah28{2c8#IHerjUS-&V{Q0#Uo#63D!Y_M4U~P7>kuaQo!sClM-~58j zr>x=C!4Jcuq$X+JKqozoVyb2XV~^Ez+{uM&eKZ$PRN$YoEu77Be(oUxM@rEAX!HTX zzh$Lt&CzgSz(k{g;<)R!PkBZw01zm~HPc4JwUU*xFO!A8y!*S6?83)F1}#s&6Bdw5BKuywwd(WB_Ze$C2x!kNBcWJ;-v6 zWseEF@18tw;fm(CRc!#fp?&IX{)w1(&c4K(=Lks%q&^b~NEYE&#-!C^9{~9*@qA+7 zU49J=x#i99VaMvbHocPsR-)ld=XsIGuZ0K{w3wdt^kl64c}b%4`V^!TXJ4K222DIUp@pmq&%7TcAssAj1haCd7 zRY__9g360Ghk5LUOegOU^oErb(7g|Jmm5dU-GkGW)U=ZZN2A^X2!cVjxq;Q_#d_O6 z{M~x^gIHJ@)xk>tc5Q4}UY7jQh^xqqAJ;6kAb&uphsX86N2dobE$;>5vWAZtn*n63 z@$~@|G|eGVhvSDy_g zHb+3i7Ux4vltx~)e0Z33#9j^=K0SU5yOp$dy=ER@rnh(hb0Sr&3Sc1#Cmw-%v<5Go zVOa^PIxhcS-vpe({2!NF=(8dGeT8M|;!wl_Vcz$idxZGVxSUxcCfL@>VYd=)R7keB zIB@d&)FR8>SODP_DiLMYtOq^El0-tF$pgDs&uv$;j!zn@ipd7^=Vwlqj%{Zfx*`EZP7sM2inimX&q7z^ZMaiM@ny|5@5N#mz5X2HE5_a0RfF5^y)HPaVcrzuAPn4ec5qfk5MRnp8x1j}8VVkvh3s>-cV3vd8G1zSZ!l<@LI(g4+mPO6I zE>?%{sWBCc^lDoZ6sQv$?AwU%IU^2f>r*n`!%FZ}Db6Qf%T+DR;EbMj6YK{!c8E#-t^u}a7LI#W zDB$a+-_GL-J8NdKc*I^zVJ#qbRk4WIkFh*@);o9KJ%%+*Mg1oxUj=sJ`gWZ(UN1IP zFq{^VGNFU{BeL@Z796)X{Lwz~?T)$4pfFA>p`!(c0Uk2KWfAoQuY&Jn7n6UQ?F>=h zPI|YO)8Z?tJ-^$fc=!@jRMhfOsavm!(Y+?_SokVal`+7BM}jJ-*rJ}tg7Y3IL(9Fk zaZ0jLh`~AGy5IRRSD&(dEoly%s-dUU4>=>k1xP4hU^!M`84r-yJl{TV^b>phu~Ub! z&OARl3r7~NGm*d+!W7uB$20dYipJLE#QwDJ5fA3lQn^(b{(^=yz8MNwkDkORE$C1n zbkk);Q_q?|x&-tbd?hz5)5IlSCZN>M5(tg&bPIs@xC*+fi&JGed#U(8isLraF|t+@ z$pt&-auNf}+Yf**JT<7+>8!5}LPQxX$r;!Ra4f%RK?~W-Bi)X3fLtegQ^xhXiUvL6 z4e(l!!#Y$PLr&Vn=6vqPE_2=sgpn?J>&%%= zL%y3*WO4h&V^{bJgOt#)6}_%k=$+zo>u)Cyc^`L{Bkz@&Kbb+cs&u@HnOppJ7pb%4 ztv!#>a^cuowjXW~>s?0Re_L6zFnFZJNEg^!86Tcf5QL3t$IHAofCcP9JQ=R>^-!Qm z&3p1)5yRp0b1|7TGisTgFIf+y?*=zb(-)VXQ1))^gpg$s*?$kK%($#NSrLS1O!$Hl@LP2Vi0 zVro)*py6NpH)@P@usugzfjVz41aE>x6Ez6&+fKN?b$R2>&u8%6mv1cOE@?MXJ~ftT zN_G35rACsmJEisvGKxjmvj( za$XdmOl|(7byPrGBKWD5L1y@CWZ5z~$*<-5V)>78$b7lwz^?|v&4TSd%ub+{+y~lc&s>fE@NWY(@ z3&l!$ekh#vMzYniKvgvg-n)sLh<`8elN~8BV(Pg6rfk;z-Lk~CqbSsx$3AbO;}kRb znoE3ru0o+4%~vOu_Z}8osFjcc5F>@Qx1gWKB2f1?bZ)QEVXR+hWG42P*(^oDclU1U+b-S*t;VvjcK2W z&8-y3$rw{Ck_X8+_NSLz?!4O)`DD@Z*XLB-58Qyyjgs`o4@+E0?T1u+UeKAyB;L*7 zw#qtAbHivhz*5>4LZUf8AD*7;6ZQrdKb32t9!%AgxN^}6u8+6_r;UdYb}p~mNVr#! zajuzk8$RtrOld_BGl(-}DN`S~}bor%j2_Tn*n@B|r+ zfw9G&xL<7gr<9zXtAFib?s6(t5JX{y7m@WxddqhHcev3^+1_vr!^}66zXQ4+?Uv6X zmYXKCisQau>9==!Fu8o18=a;^y8kCG>g!joIHOqj`|F!{zU8oT!>kI`&pk(HPBT%j z{F(68dnfvf&HwH7RSjU5=Cvh@#2&iWb1+%JY;~vLv!p46?A_)p0(-ZMZC}m81MpixQ3techJo*rg*tz;hGRwqC-*x1CvjnJsR_9fM{ z8`26v^q=E_`L8WU??Pq>oND8tFPUQ|q1*5cZnJ8o2n%nHtJQV58KaAIdeRut*njse z$RiivNd4vUtg&C<`H{bZ>^|L8PuD{qza)?2HiLZabF`EQ9W-*Ym(=D?-P#UQ;hd8&+@X4;fe zT{NBT&1?x=_P$;KcVGU595_;ZK^_Y@ZkH~@_&9>}Zp2e^THi4THK5+zg{aqR$|bQf zlsNaP0u;qfcwMV64})EyO`S6kGxsmQWB#xkoD>y6Uc2qc>MJbiLGzO8=WJ7A#K!Hv zhWpE=Um~-Mx#h!&b>|)j6nr4SwK}i%aU%U?#>Hk}d`t_>hc9rUkL(syf{aXmuSt){ z3EPMZBJkoh_Sv;^eA%n$6L_T&sC`KThxC!xn7)-11R8$a<21K=m+yDh7iotPc_ z1D>fW9(4gf5gMQ>0n2Le@wGVI4a%5U`2v^Sg6xgJuyV;9^Fx5~uK&=u{$3at-um|_ zsv0k29ynhrj0aCI6wt4mVd@LIA2qlWEx<9(Eb{06UO=5|R_l$VQ?I8GV~@wYmIg%} z`J~^t4cSf-^t^B|;^Z~dy9XGm`QwDU{o>0Oyej8syKFitv< z0)JF7ov&6;eZn@9HQ<5|Q!gdK6qdP}LC#E#ZqxXcd>8mYcPG4oVb>$~Jjaa`BUP2z z)(3Ua>9VZt>U{wfB#heTZuAAhJzYx93d!f_;Az4n+|tBff{pG{f?Jr-c6v9-?`n-k zTL*k(th)s%P?=x%(6IFveY-tJ*`3Juy?e=4fOEjA=fpl;)a1Zm!#5Rv7f`sjnBMRU z8+F6TbZuwJdtB7dU$zwzHg)y9XpDOm%OL>Vc_}Jjp<`SNy#b>v)lab`BKq#BDC$L; zJD0_f?ck!`xuzsU56BGG`Kj2H&y9kVh*6RKT2QTuj>`M#O~0uYu)SnU-g?G3(ZCAr zOx;wbT!w7FyY+F)go2NV(B~SVU-gZLd3n*ik1yA0MItql z7izlFnx24HV!Fryuk5a|GBsEpIrChl9bqK4i==NVl6Q|bEdpa)zFgG1^#EUUC=9Y@ z5PR{LD5}__kfL-zOO0!eaUwuK&FO;I%|(4ozlOUC08o89A)FR9`E?W9MV)kBV_-uK zMPk$!*LMA#ACjC%Z6Gn0oKZ_VVAFqp{}x74O?Qmc07_%J}WK0MLvl=d+SjI0WA9M`1?NJ?Kf;4xND&+_Y8;m zjjCst%gv*Xl=w&iv(iW;IBM`v6DfC)bPjh3pmd%ZEENay;+kedhK33DSb$)>zNpn(7K`JjP+p z?Lo_`Clz<4Na%H?x`MmsFD8^s@$-c!d5h4=PMnrtBNsR~*d_GBZGJF}gh#J(%IT;Z zL`bh4e@59=JQXomd{$hqxQEU%`EcMd)k-VD}&q*V#YX9OR(1D!+-ke@-_oLSB-TrxaSRe#0a>4Um~R+Xsf&1{Mt!J0xmy;;NeK4&{R(~k8G4s8BCUwD=hT1Q5tU1fbBob;5%K8w=UgtzjIW$j4Tv)`?N^!hR*EtF-aS7oo! z7T|0gV;Uv!^)j7MB43K91Ja`uvMVX%ASK>xg`hiSc~3Ai0U!?PcZ@>E3=c9-y3X3Y z-i67BdR{Gd+*H6db7MktdT2A%RG5HwkE1A3pTS8@DK8QPkl{+V@)cN#M65CY^@Gg@}D##$ob&6M|>qI2wfr%FMzfnO)iSGgY?@v zyxcq1;U+B+C|$d{CsbQ4FnOEVg)ni3j`U%+pmF<~?SE_?dVQYg1P-2D q{LyJ(_S!`{;r~|P_y6SD>yt2q?MdW2>6pg< + + diff --git a/docker/monitoring/configuration/dashboards.yaml b/docker/monitoring/configuration/dashboards.yaml new file mode 100644 index 0000000..e59ac96 --- /dev/null +++ b/docker/monitoring/configuration/dashboards.yaml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: +- name: 'Prometheus' + orgId: 1 + folder: '' + type: file + options: + path: /var/lib/grafana/dashboards \ No newline at end of file diff --git a/docker/monitoring/configuration/dashboards/waku-analysis.json b/docker/monitoring/configuration/dashboards/waku-analysis.json new file mode 100644 index 0000000..395c6ec --- /dev/null +++ b/docker/monitoring/configuration/dashboards/waku-analysis.json @@ -0,0 +1,3416 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Metrics for Waku nodes written in Nim", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 36, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 56, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "At a glance", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 150, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0 + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "green", + "value": 2 + }, + { + "color": "#EAB839", + "value": 120 + }, + { + "color": "red", + "value": 149 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 52, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "libp2p_pubsub_peers{instance=~\"[[instance]]\"}", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Libp2p PubSub Peers", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "dateTimeAsIso" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 46, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "titleSize": 18, + "valueSize": 20 + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "process_start_time_seconds{instance=~\"[[instance]]\"} * 1000", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Node start times (UTC)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 78, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "(increase(libp2p_network_bytes_total{instance=~\"[[instance]]\", direction=\"in\"}[15m]))/1000000/(900)*8", + "interval": "", + "legendFormat": "{{instance}}:{{direction}}", + "range": true, + "refId": "A" + } + ], + "title": "Inbound Traffic (Mbps)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 79, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "(increase(libp2p_network_bytes_total{instance=~\"[[instance]]\", direction=\"out\"}[15m]))/1000000/(900)*8", + "interval": "", + "legendFormat": "{{instance}}:{{direction}}", + "range": true, + "refId": "A" + } + ], + "title": "Outbound Traffic (Mbps)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "relay: node-01.do-ams3.status.prod" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "(increase(waku_node_messages_total{instance=~\"[[instance]]\"}[1m]))", + "interval": "", + "legendFormat": "{{type}}: {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Messages (1m rate)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 9, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (type)(increase(waku_peers_errors{instance=~\"[[instance]]\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "peer {{type}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (type)(increase(waku_store_errors{instance=~\"[[instance]]\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "store {{type}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (type)(increase(waku_node_errors{instance=~\"[[instance]]\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "legendFormat": "node {{type}}", + "range": true, + "refId": "C" + } + ], + "title": "Waku Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "id": 66, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "count(count by (contentTopic)(waku_node_messages_total))", + "interval": "", + "legendFormat": "content topics", + "refId": "A" + } + ], + "title": "Total Content Topics", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepBefore", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "id": 68, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "waku_version{instance=~\"[[instance]]\"}", + "interval": "", + "legendFormat": "{{instance}}:{{version}}", + "range": true, + "refId": "A" + } + ], + "title": "Waku version", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 17, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "General", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 48, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "waku_node_filters{instance=~\"[[instance]]\"}", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Waku Node Filters", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 50, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "waku_node_errors{instance=~\"[[instance]]\"}", + "interval": "", + "legendFormat": "{{type}}: {{instance}}", + "refId": "A" + } + ], + "title": "Waku Node Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "libp2p_pubsub_topics {instance=~\"[[instance]]\"}", + "interval": "", + "legendFormat": "Topics: {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "libp2p_pubsub_subscriptions_total {instance=~\"[[instance]]\"}", + "hide": false, + "interval": "", + "legendFormat": "Subscriptions: {{instance}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "libp2p_pubsub_unsubscriptions_total {instance=~\"[[instance]]\"}", + "hide": false, + "interval": "", + "legendFormat": "Unsubscriptions: {{instance}}", + "refId": "C" + } + ], + "title": "Pubsub Topics", + "type": "timeseries" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 44 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum by (instance)(libp2p_pubsub_peers{instance=~\"[[instance]]\"})", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "LibP2P PubSub Peers", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1232", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1233", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 5, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 50 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum by (instance)(libp2p_peers{instance=~\"[[instance]]\"})", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "LibP2P Peers", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:1306", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:1307", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 50 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "libp2p_pubsub_validation_success_total{instance=~\"[[instance]]\"}", + "hide": false, + "interval": "", + "legendFormat": "success {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "libp2p_pubsub_validation_failure_total{instance=~\"[[instance]]\"}", + "hide": false, + "interval": "", + "legendFormat": "failure {{instance}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "libp2p_pubsub_validation_ignore_total{instance=~\"[[instance]]\"}", + "hide": false, + "interval": "", + "legendFormat": "ignore {{instance}}", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "LibP2P Validations", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:189", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:190", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 56 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum by (type)(libp2p_open_streams{instance=~\"[[instance]]\"})", + "interval": "", + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "LibP2P Open Streams", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:115", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:116", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 56 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "libp2p_total_dial_attempts_total{instance=~\"[[instance]]\"}", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "Attempts: {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "libp2p_failed_dials_total{instance=~\"[[instance]]\"}", + "hide": false, + "interval": "", + "legendFormat": "Failed: {{instance}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "libp2p_successful_dials_total{instance=~\"[[instance]]\"}", + "hide": false, + "interval": "", + "legendFormat": "Successful: {{instance}}", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "LibP2P Dials", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:189", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:190", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 62 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum by (instance)(process_open_fds{instance=~\"[[instance]]\"})", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Open File Descriptors", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:511", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:512", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 4, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 68 + }, + "id": 44, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "nim_gc_mem_bytes{instance=~\"[[instance]]\"}", + "interval": "", + "legendFormat": "Nim total memory: {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "nim_gc_mem_occupied_bytes{instance=~\"[[instance]]\"}", + "hide": false, + "interval": "", + "legendFormat": "Nim occupied memory: {{instance}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "nim_gc_heap_instance_occupied_summed_bytes{instance=~\"[[instance]]\"}", + "hide": false, + "interval": "", + "legendFormat": "Nim total heap: {{instance}}", + "refId": "C" + } + ], + "title": "Nim Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 68 + }, + "id": 64, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "nim_gc_heap_instance_occupied_bytes{instance=~\"[[instance]]\"}", + "interval": "", + "legendFormat": "{{instance}} {{type_name}}", + "refId": "A" + } + ], + "title": "Heap allocation", + "type": "timeseries" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 76 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum by (instance)(process_virtual_memory_bytes{instance=~\"[[instance]]\"})", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Virtual Memory", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:263", + "format": "decbytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:264", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 76 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.2.5", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "sum by (instance)(process_resident_memory_bytes{instance=~\"[[instance]]\"})", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Resident Memory", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:437", + "format": "decbytes", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:438", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 3, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 83 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "increase(waku_bridge_transfers_total{instance=~\"[[instance]]\"}[10m])", + "interval": "", + "legendFormat": "{{fleet}} : {{type}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "increase(envelopes_valid_total{instance=~\"bridge*[[instance]]\"}[10m])", + "hide": false, + "interval": "", + "legendFormat": "{{fleet}} : v1_envelopes", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "increase(waku_node_messages_total{instance=~\"bridge*[[instance]]\"}[10m])", + "hide": false, + "interval": "", + "legendFormat": "{{fleet}} : v2_messages", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "increase(envelopes_dropped_total{instance=~\"bridge*[[instance]]\"}[10m])", + "hide": false, + "interval": "", + "legendFormat": "{{fleet}} : v1_envelopes_dropped ({{reason}})", + "refId": "D" + } + ], + "title": "Bridge (10m rate)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 83 + }, + "id": 74, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "connected_peers{instance=~\"bridge*[[instance]]\"}", + "interval": "", + "legendFormat": "v1_connected_peers", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "libp2p_pubsub_peers{instance=~\"bridge*[[instance]]\"}", + "hide": false, + "interval": "", + "legendFormat": "v2_connected_peers", + "refId": "B" + } + ], + "title": "Connected Peers", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 91 + }, + "id": 34, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "Store/Archive", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 3, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 98 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "increase(waku_store_queries{instance=~\"nwaku_postgres:8003\"}[1m])", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Waku Store Queries (Postgres) (1m rate)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 3, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 98 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "increase(waku_store_queries{instance=~\"nwaku_sqlite:8004\"}[1m])", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Waku Store Queries (SQLite) (1m rate)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 98 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (type)(increase(waku_archive_errors{instance=~\"nwaku_postgres:8003\"}[1m]))", + "hide": false, + "interval": "", + "legendFormat": "{{type}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (type)(increase(waku_store_errors{instance=~\"nwaku_postgres:8003\"}[1m]))", + "hide": false, + "interval": "", + "legendFormat": "{{type}}", + "range": true, + "refId": "B" + } + ], + "title": "Waku Archive Errors (Postgres) (1m rate)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 98 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (type)(increase(waku_archive_errors{instance=~\"nwaku_:8004\"}[1m]))", + "hide": false, + "interval": "", + "legendFormat": "{{type}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (type)(increase(waku_store_errors{instance=~\"nwaku_sqlite:8004\"}[1m]))", + "hide": false, + "interval": "", + "legendFormat": "{{type}}", + "range": true, + "refId": "B" + } + ], + "title": "Waku Archive Errors (SQLite) (1m rate)", + "type": "timeseries" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 104 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 77, + "legend": { + "show": false + }, + "maxDataPoints": 60, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "9.2.5", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(increase(waku_archive_query_duration_seconds_bucket{instance=~\"nwaku_postgres:8003\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(increase(waku_store_query_duration_seconds_bucket{instance=~\"nwaku_postgres:8003\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": true, + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Waku Archive Query Duration (Postgres)", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 104 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 77, + "legend": { + "show": false + }, + "maxDataPoints": 60, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "9.2.5", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(increase(waku_archive_query_duration_seconds_bucket{instance=~\"nwaku_sqlite:8004\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(increase(waku_store_query_duration_seconds_bucket{instance=~\"nwaku_sqlite:8004\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": true, + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Waku Archive Query Duration (SQLite)", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 104 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 75, + "legend": { + "show": false + }, + "maxDataPoints": 60, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "9.2.5", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(waku_archive_insert_duration_seconds_bucket{instance=~\"nwaku_postgres:8003\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(waku_store_insert_duration_seconds_bucket{instance=~\"nwaku_postgres:8003\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Waku Archive Insert Duration (Postgres)", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 104 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 75, + "legend": { + "show": false + }, + "maxDataPoints": 60, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "9.2.5", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(waku_archive_insert_duration_seconds_bucket{instance=~\"nwaku_sqlite:8004\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(waku_store_insert_duration_seconds_bucket{instance=~\"nwaku_sqlite:8004\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "title": "Waku Archive Insert Duration (SQLite)", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + } + ], + "refresh": "5s", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "nwaku_.*", + "value": "nwaku_.*" + }, + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [ + { + "selected": true, + "text": "nwaku_postgres", + "value": "nwaku_postgres:8003" + }, + { + "selected": true, + "text": "nwaku_sqlite", + "value": "nwaku_sqlite:8004" + } + ], + "query": "nwaku_.*", + "queryValue": "nwaku_.*", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-10d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "browser", + "title": "Waku Analysis", + "uid": "qrp_ZCTGz", + "version": 104, + "weekStart": "" +} diff --git a/docker/monitoring/configuration/datasources.yaml b/docker/monitoring/configuration/datasources.yaml new file mode 100644 index 0000000..9f4f51f --- /dev/null +++ b/docker/monitoring/configuration/datasources.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + org_id: 1 + url: http://prometheus:9090 + is_default: true + version: 1 + editable: true \ No newline at end of file diff --git a/docker/monitoring/configuration/grafana-plugins.env b/docker/monitoring/configuration/grafana-plugins.env new file mode 100644 index 0000000..2a4b487 --- /dev/null +++ b/docker/monitoring/configuration/grafana-plugins.env @@ -0,0 +1 @@ +GF_INSTALL_PLUGINS=grafana-worldmap-panel,grafana-piechart-panel,yesoreyeram-boomtheme-panel,briangann-gauge-panel,pierosavi-imageit-panel,bessler-pictureit-panel,vonage-status-panel diff --git a/docker/monitoring/configuration/grafana.ini b/docker/monitoring/configuration/grafana.ini new file mode 100644 index 0000000..f237726 --- /dev/null +++ b/docker/monitoring/configuration/grafana.ini @@ -0,0 +1,51 @@ +instance_name = nwaku dashboard + +;[dashboards.json] +;enabled = true +;path = /home/git/grafana/grafana-dashboards/dashboards + + +#################################### Auth ########################## +[auth] +disable_login_form = false + +#################################### Anonymous Auth ########################## +[auth.anonymous] +# enable anonymous access +enabled = true + +# specify organization name that should be used for unauthenticated users +;org_name = Public + +# specify role for unauthenticated users +org_role = Admin +; org_role = Viewer + +;[security] +;admin_user = ocr +;admin_password = ocr + +;[users] +# disable user signup / registration +;allow_sign_up = false + +# Set to true to automatically assign new users to the default organization (id 1) +;auto_assign_org = true + +# Default role new users will be automatically assigned (if disabled above is set to true) +;auto_assign_org_role = Viewer + +#################################### SMTP / Emailing ########################## +;[smtp] +;enabled = false +;host = localhost:25 +;user = +;password = +;cert_file = +;key_file = +;skip_verify = false +;from_address = admin@grafana.localhost + +;[emails] +;welcome_email_on_sign_up = false + diff --git a/docker/monitoring/configuration/pg-exporter-queries.yml b/docker/monitoring/configuration/pg-exporter-queries.yml new file mode 100644 index 0000000..38234f2 --- /dev/null +++ b/docker/monitoring/configuration/pg-exporter-queries.yml @@ -0,0 +1,263 @@ +pg_replication: + query: "SELECT CASE WHEN NOT pg_is_in_recovery() THEN 0 ELSE GREATEST (0, EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp()))) END AS lag" + master: true + metrics: + - lag: + usage: "GAUGE" + description: "Replication lag behind master in seconds" + +pg_postmaster: + query: "SELECT pg_postmaster_start_time as start_time_seconds from pg_postmaster_start_time()" + master: true + metrics: + - start_time_seconds: + usage: "GAUGE" + description: "Time at which postmaster started" + +pg_stat_user_tables: + query: | + SELECT + current_database() datname, + schemaname, + relname, + seq_scan, + seq_tup_read, + idx_scan, + idx_tup_fetch, + n_tup_ins, + n_tup_upd, + n_tup_del, + n_tup_hot_upd, + n_live_tup, + n_dead_tup, + n_mod_since_analyze, + COALESCE(last_vacuum, '1970-01-01Z') as last_vacuum, + COALESCE(last_autovacuum, '1970-01-01Z') as last_autovacuum, + COALESCE(last_analyze, '1970-01-01Z') as last_analyze, + COALESCE(last_autoanalyze, '1970-01-01Z') as last_autoanalyze, + vacuum_count, + autovacuum_count, + analyze_count, + autoanalyze_count + FROM + pg_stat_user_tables + metrics: + - datname: + usage: "LABEL" + description: "Name of current database" + - schemaname: + usage: "LABEL" + description: "Name of the schema that this table is in" + - relname: + usage: "LABEL" + description: "Name of this table" + - seq_scan: + usage: "COUNTER" + description: "Number of sequential scans initiated on this table" + - seq_tup_read: + usage: "COUNTER" + description: "Number of live rows fetched by sequential scans" + - idx_scan: + usage: "COUNTER" + description: "Number of index scans initiated on this table" + - idx_tup_fetch: + usage: "COUNTER" + description: "Number of live rows fetched by index scans" + - n_tup_ins: + usage: "COUNTER" + description: "Number of rows inserted" + - n_tup_upd: + usage: "COUNTER" + description: "Number of rows updated" + - n_tup_del: + usage: "COUNTER" + description: "Number of rows deleted" + - n_tup_hot_upd: + usage: "COUNTER" + description: "Number of rows HOT updated (i.e., with no separate index update required)" + - n_live_tup: + usage: "GAUGE" + description: "Estimated number of live rows" + - n_dead_tup: + usage: "GAUGE" + description: "Estimated number of dead rows" + - n_mod_since_analyze: + usage: "GAUGE" + description: "Estimated number of rows changed since last analyze" + - last_vacuum: + usage: "GAUGE" + description: "Last time at which this table was manually vacuumed (not counting VACUUM FULL)" + - last_autovacuum: + usage: "GAUGE" + description: "Last time at which this table was vacuumed by the autovacuum daemon" + - last_analyze: + usage: "GAUGE" + description: "Last time at which this table was manually analyzed" + - last_autoanalyze: + usage: "GAUGE" + description: "Last time at which this table was analyzed by the autovacuum daemon" + - vacuum_count: + usage: "COUNTER" + description: "Number of times this table has been manually vacuumed (not counting VACUUM FULL)" + - autovacuum_count: + usage: "COUNTER" + description: "Number of times this table has been vacuumed by the autovacuum daemon" + - analyze_count: + usage: "COUNTER" + description: "Number of times this table has been manually analyzed" + - autoanalyze_count: + usage: "COUNTER" + description: "Number of times this table has been analyzed by the autovacuum daemon" + +pg_statio_user_tables: + query: "SELECT current_database() datname, schemaname, relname, heap_blks_read, heap_blks_hit, idx_blks_read, idx_blks_hit, toast_blks_read, toast_blks_hit, tidx_blks_read, tidx_blks_hit FROM pg_statio_user_tables" + metrics: + - datname: + usage: "LABEL" + description: "Name of current database" + - schemaname: + usage: "LABEL" + description: "Name of the schema that this table is in" + - relname: + usage: "LABEL" + description: "Name of this table" + - heap_blks_read: + usage: "COUNTER" + description: "Number of disk blocks read from this table" + - heap_blks_hit: + usage: "COUNTER" + description: "Number of buffer hits in this table" + - idx_blks_read: + usage: "COUNTER" + description: "Number of disk blocks read from all indexes on this table" + - idx_blks_hit: + usage: "COUNTER" + description: "Number of buffer hits in all indexes on this table" + - toast_blks_read: + usage: "COUNTER" + description: "Number of disk blocks read from this table's TOAST table (if any)" + - toast_blks_hit: + usage: "COUNTER" + description: "Number of buffer hits in this table's TOAST table (if any)" + - tidx_blks_read: + usage: "COUNTER" + description: "Number of disk blocks read from this table's TOAST table indexes (if any)" + - tidx_blks_hit: + usage: "COUNTER" + description: "Number of buffer hits in this table's TOAST table indexes (if any)" + +# WARNING: This set of metrics can be very expensive on a busy server as every unique query executed will create an additional time series +pg_stat_statements: + query: "SELECT t2.rolname, t3.datname, queryid, calls, ( total_plan_time + total_exec_time ) / 1000 as total_time_seconds, ( min_plan_time + min_exec_time ) / 1000 as min_time_seconds, ( max_plan_time + max_exec_time ) / 1000 as max_time_seconds, ( mean_plan_time + mean_exec_time ) / 1000 as mean_time_seconds, ( stddev_plan_time + stddev_exec_time ) / 1000 as stddev_time_seconds, rows, shared_blks_hit, shared_blks_read, shared_blks_dirtied, shared_blks_written, local_blks_hit, local_blks_read, local_blks_dirtied, local_blks_written, temp_blks_read, temp_blks_written, blk_read_time / 1000 as blk_read_time_seconds, blk_write_time / 1000 as blk_write_time_seconds FROM pg_stat_statements t1 JOIN pg_roles t2 ON (t1.userid=t2.oid) JOIN pg_database t3 ON (t1.dbid=t3.oid) WHERE t2.rolname != 'rdsadmin' AND queryid IS NOT NULL" + master: true + metrics: + - rolname: + usage: "LABEL" + description: "Name of user" + - datname: + usage: "LABEL" + description: "Name of database" + - queryid: + usage: "LABEL" + description: "Query ID" + - calls: + usage: "COUNTER" + description: "Number of times executed" + - total_time_seconds: + usage: "COUNTER" + description: "Total time spent in the statement, in milliseconds" + - min_time_seconds: + usage: "GAUGE" + description: "Minimum time spent in the statement, in milliseconds" + - max_time_seconds: + usage: "GAUGE" + description: "Maximum time spent in the statement, in milliseconds" + - mean_time_seconds: + usage: "GAUGE" + description: "Mean time spent in the statement, in milliseconds" + - stddev_time_seconds: + usage: "GAUGE" + description: "Population standard deviation of time spent in the statement, in milliseconds" + - rows: + usage: "COUNTER" + description: "Total number of rows retrieved or affected by the statement" + - shared_blks_hit: + usage: "COUNTER" + description: "Total number of shared block cache hits by the statement" + - shared_blks_read: + usage: "COUNTER" + description: "Total number of shared blocks read by the statement" + - shared_blks_dirtied: + usage: "COUNTER" + description: "Total number of shared blocks dirtied by the statement" + - shared_blks_written: + usage: "COUNTER" + description: "Total number of shared blocks written by the statement" + - local_blks_hit: + usage: "COUNTER" + description: "Total number of local block cache hits by the statement" + - local_blks_read: + usage: "COUNTER" + description: "Total number of local blocks read by the statement" + - local_blks_dirtied: + usage: "COUNTER" + description: "Total number of local blocks dirtied by the statement" + - local_blks_written: + usage: "COUNTER" + description: "Total number of local blocks written by the statement" + - temp_blks_read: + usage: "COUNTER" + description: "Total number of temp blocks read by the statement" + - temp_blks_written: + usage: "COUNTER" + description: "Total number of temp blocks written by the statement" + - blk_read_time_seconds: + usage: "COUNTER" + description: "Total time the statement spent reading blocks, in milliseconds (if track_io_timing is enabled, otherwise zero)" + - blk_write_time_seconds: + usage: "COUNTER" + description: "Total time the statement spent writing blocks, in milliseconds (if track_io_timing is enabled, otherwise zero)" + +pg_process_idle: + query: | + WITH + metrics AS ( + SELECT + application_name, + SUM(EXTRACT(EPOCH FROM (CURRENT_TIMESTAMP - state_change))::bigint)::float AS process_idle_seconds_sum, + COUNT(*) AS process_idle_seconds_count + FROM pg_stat_activity + WHERE state = 'idle' + GROUP BY application_name + ), + buckets AS ( + SELECT + application_name, + le, + SUM( + CASE WHEN EXTRACT(EPOCH FROM (CURRENT_TIMESTAMP - state_change)) <= le + THEN 1 + ELSE 0 + END + )::bigint AS bucket + FROM + pg_stat_activity, + UNNEST(ARRAY[1, 2, 5, 15, 30, 60, 90, 120, 300]) AS le + GROUP BY application_name, le + ORDER BY application_name, le + ) + SELECT + application_name, + process_idle_seconds_sum as seconds_sum, + process_idle_seconds_count as seconds_count, + ARRAY_AGG(le) AS seconds, + ARRAY_AGG(bucket) AS seconds_bucket + FROM metrics JOIN buckets USING (application_name) + GROUP BY 1, 2, 3 + metrics: + - application_name: + usage: "LABEL" + description: "Application Name" + - seconds: + usage: "HISTOGRAM" + description: "Idle time of server processes" diff --git a/docker/monitoring/configuration/postgres-exporter.yml b/docker/monitoring/configuration/postgres-exporter.yml new file mode 100644 index 0000000..a8380dd --- /dev/null +++ b/docker/monitoring/configuration/postgres-exporter.yml @@ -0,0 +1,9 @@ +auth_modules: + mypostgres: + type: userpass + userpass: + username: postgres + password: ${POSTGRES_PASSWORD} + options: + # options become key=value parameters of the DSN + sslmode: disable diff --git a/docker/monitoring/prometheus-config.yml b/docker/monitoring/prometheus-config.yml new file mode 100644 index 0000000..e4398cc --- /dev/null +++ b/docker/monitoring/prometheus-config.yml @@ -0,0 +1,18 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + monitor: "Monitoring" + +scrape_configs: + - job_name: "nwaku_postgres" + static_configs: + - targets: ['nwaku_postgres:8003'] + + - job_name: "nwaku_sqlite" + static_configs: + - targets: ['nwaku_sqlite:8004'] + + - job_name: postgres-exporter + static_configs: + - targets: ['postgres-exporter:9187'] diff --git a/docker/msg_publisher.sh b/docker/msg_publisher.sh new file mode 100644 index 0000000..8d067a3 --- /dev/null +++ b/docker/msg_publisher.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +## Install the `dig` command +apk --no-cache add bind-tools + +apk --no-cache add curl + +## Install coreutils so that the `date` command behaves as the GNU one +apk --no-cache add coreutils + +## The port numbers and host names are defined in the 'docker-compose.yml' file +nwaku_postgres_IP=$(dig +short nwaku_postgres) +nwaku_sqlite_IP=$(dig +short nwaku_sqlite) +target_postgres="http://${nwaku_postgres_IP}:8545" +target_sqlite="http://${nwaku_sqlite_IP}:8546" + +echo "This is publisher: ${target_postgres}; ${target_sqlite}" + +## Wait a few seconds until the `nwaku` nodes started their rpc services +sleep 20 + +while true +do + ## Make the ""postgres"" node to publish a message + curl -d "{\"jsonrpc\":\"2.0\",\"id\":"$(date +%s%N)",\"method\":\"post_waku_v2_relay_v1_message\", \"params\":[\"/waku/2/default-waku/proto\", {\"timestamp\":"$(date +%s%N)", \"payload\":\"aGhvbGFzYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhISAK\"}]}" --header "Content-Type: application/json" ${target_postgres} + + ## Make the ""sqlite"" node to publish a message + curl -d "{\"jsonrpc\":\"2.0\",\"id\":"$(date +%s%N)",\"method\":\"post_waku_v2_relay_v1_message\", \"params\":[\"/waku/2/default-waku/proto\", {\"timestamp\":"$(date +%s%N)", \"payload\":\"aGhvbGFzYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhISAK\"}]}" --header "Content-Type: application/json" ${target_sqlite} + + sleep 0.01 +done diff --git a/docker/postgres_cfg/db.sql b/docker/postgres_cfg/db.sql new file mode 100644 index 0000000..48b8c24 --- /dev/null +++ b/docker/postgres_cfg/db.sql @@ -0,0 +1,8 @@ + +/* +The next two lines allow 'pg_stat_statements_calls' information queried +by Prometheus/Grafana. This setting is aimed for performance +analysis. Comment these lines for Prod environments. +*/ +CREATE EXTENSION pg_stat_statements; +alter system set shared_preload_libraries='pg_stat_statements'; diff --git a/docker/postgres_cfg/postgresql.conf b/docker/postgres_cfg/postgresql.conf new file mode 100644 index 0000000..59cb905 --- /dev/null +++ b/docker/postgres_cfg/postgresql.conf @@ -0,0 +1,813 @@ +# ----------------------------- +# PostgreSQL configuration file +# ----------------------------- +# +# This file consists of lines of the form: +# +# name = value +# +# (The "=" is optional.) Whitespace may be used. Comments are introduced with +# "#" anywhere on a line. The complete list of parameter names and allowed +# values can be found in the PostgreSQL documentation. +# +# The commented-out settings shown in this file represent the default values. +# Re-commenting a setting is NOT sufficient to revert it to the default value; +# you need to reload the server. +# +# This file is read on server startup and when the server receives a SIGHUP +# signal. If you edit the file on a running system, you have to SIGHUP the +# server for the changes to take effect, run "pg_ctl reload", or execute +# "SELECT pg_reload_conf()". Some parameters, which are marked below, +# require a server shutdown and restart to take effect. +# +# Any parameter can also be given as a command-line option to the server, e.g., +# "postgres -c log_connections=on". Some parameters can be changed at run time +# with the "SET" SQL command. +# +# Memory units: B = bytes Time units: us = microseconds +# kB = kilobytes ms = milliseconds +# MB = megabytes s = seconds +# GB = gigabytes min = minutes +# TB = terabytes h = hours +# d = days + + +#------------------------------------------------------------------------------ +# FILE LOCATIONS +#------------------------------------------------------------------------------ + +# The default values of these variables are driven from the -D command-line +# option or PGDATA environment variable, represented here as ConfigDir. + +#data_directory = 'ConfigDir' # use data in another directory + # (change requires restart) +#hba_file = 'ConfigDir/pg_hba.conf' # host-based authentication file + # (change requires restart) +#ident_file = 'ConfigDir/pg_ident.conf' # ident configuration file + # (change requires restart) + +# If external_pid_file is not explicitly set, no extra PID file is written. +#external_pid_file = '' # write an extra PID file + # (change requires restart) + + +#------------------------------------------------------------------------------ +# CONNECTIONS AND AUTHENTICATION +#------------------------------------------------------------------------------ + +# - Connection Settings - + +listen_addresses = '*' + # comma-separated list of addresses; + # defaults to 'localhost'; use '*' for all + # (change requires restart) +#port = 5432 # (change requires restart) +max_connections = 100 # (change requires restart) +#superuser_reserved_connections = 3 # (change requires restart) +#unix_socket_directories = '/var/run/postgresql' # comma-separated list of directories + # (change requires restart) +#unix_socket_group = '' # (change requires restart) +#unix_socket_permissions = 0777 # begin with 0 to use octal notation + # (change requires restart) +#bonjour = off # advertise server via Bonjour + # (change requires restart) +#bonjour_name = '' # defaults to the computer name + # (change requires restart) + +# - TCP settings - +# see "man tcp" for details + +#tcp_keepalives_idle = 0 # TCP_KEEPIDLE, in seconds; + # 0 selects the system default +#tcp_keepalives_interval = 0 # TCP_KEEPINTVL, in seconds; + # 0 selects the system default +#tcp_keepalives_count = 0 # TCP_KEEPCNT; + # 0 selects the system default +#tcp_user_timeout = 0 # TCP_USER_TIMEOUT, in milliseconds; + # 0 selects the system default + +#client_connection_check_interval = 0 # time between checks for client + # disconnection while running queries; + # 0 for never + +# - Authentication - + +#authentication_timeout = 1min # 1s-600s +#password_encryption = scram-sha-256 # scram-sha-256 or md5 +#db_user_namespace = off + +# GSSAPI using Kerberos +#krb_server_keyfile = 'FILE:${sysconfdir}/krb5.keytab' +#krb_caseins_users = off + +# - SSL - + +#ssl = off +#ssl_ca_file = '' +#ssl_cert_file = 'server.crt' +#ssl_crl_file = '' +#ssl_crl_dir = '' +#ssl_key_file = 'server.key' +#ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' # allowed SSL ciphers +#ssl_prefer_server_ciphers = on +#ssl_ecdh_curve = 'prime256v1' +#ssl_min_protocol_version = 'TLSv1.2' +#ssl_max_protocol_version = '' +#ssl_dh_params_file = '' +#ssl_passphrase_command = '' +#ssl_passphrase_command_supports_reload = off + + +#------------------------------------------------------------------------------ +# RESOURCE USAGE (except WAL) +#------------------------------------------------------------------------------ + +# - Memory - + +shared_buffers = 128MB # min 128kB + # (change requires restart) +#huge_pages = try # on, off, or try + # (change requires restart) +#huge_page_size = 0 # zero for system default + # (change requires restart) +#temp_buffers = 8MB # min 800kB +#max_prepared_transactions = 0 # zero disables the feature + # (change requires restart) +# Caution: it is not advisable to set max_prepared_transactions nonzero unless +# you actively intend to use prepared transactions. +#work_mem = 4MB # min 64kB +#hash_mem_multiplier = 2.0 # 1-1000.0 multiplier on hash table work_mem +#maintenance_work_mem = 64MB # min 1MB +#autovacuum_work_mem = -1 # min 1MB, or -1 to use maintenance_work_mem +#logical_decoding_work_mem = 64MB # min 64kB +#max_stack_depth = 2MB # min 100kB +#shared_memory_type = mmap # the default is the first option + # supported by the operating system: + # mmap + # sysv + # windows + # (change requires restart) +dynamic_shared_memory_type = posix # the default is usually the first option + # supported by the operating system: + # posix + # sysv + # windows + # mmap + # (change requires restart) +#min_dynamic_shared_memory = 0MB # (change requires restart) + +# - Disk - + +#temp_file_limit = -1 # limits per-process temp file space + # in kilobytes, or -1 for no limit + +# - Kernel Resources - + +#max_files_per_process = 1000 # min 64 + # (change requires restart) + +# - Cost-Based Vacuum Delay - + +#vacuum_cost_delay = 0 # 0-100 milliseconds (0 disables) +#vacuum_cost_page_hit = 1 # 0-10000 credits +#vacuum_cost_page_miss = 2 # 0-10000 credits +#vacuum_cost_page_dirty = 20 # 0-10000 credits +#vacuum_cost_limit = 200 # 1-10000 credits + +# - Background Writer - + +#bgwriter_delay = 200ms # 10-10000ms between rounds +#bgwriter_lru_maxpages = 100 # max buffers written/round, 0 disables +#bgwriter_lru_multiplier = 2.0 # 0-10.0 multiplier on buffers scanned/round +#bgwriter_flush_after = 512kB # measured in pages, 0 disables + +# - Asynchronous Behavior - + +#backend_flush_after = 0 # measured in pages, 0 disables +#effective_io_concurrency = 1 # 1-1000; 0 disables prefetching +#maintenance_io_concurrency = 10 # 1-1000; 0 disables prefetching +#max_worker_processes = 8 # (change requires restart) +#max_parallel_workers_per_gather = 2 # taken from max_parallel_workers +#max_parallel_maintenance_workers = 2 # taken from max_parallel_workers +#max_parallel_workers = 8 # maximum number of max_worker_processes that + # can be used in parallel operations +#parallel_leader_participation = on +#old_snapshot_threshold = -1 # 1min-60d; -1 disables; 0 is immediate + # (change requires restart) + + +#------------------------------------------------------------------------------ +# WRITE-AHEAD LOG +#------------------------------------------------------------------------------ + +# - Settings - + +#wal_level = replica # minimal, replica, or logical + # (change requires restart) +#fsync = on # flush data to disk for crash safety + # (turning this off can cause + # unrecoverable data corruption) +#synchronous_commit = on # synchronization level; + # off, local, remote_write, remote_apply, or on +#wal_sync_method = fsync # the default is the first option + # supported by the operating system: + # open_datasync + # fdatasync (default on Linux and FreeBSD) + # fsync + # fsync_writethrough + # open_sync +#full_page_writes = on # recover from partial page writes +#wal_log_hints = off # also do full page writes of non-critical updates + # (change requires restart) +#wal_compression = off # enables compression of full-page writes; + # off, pglz, lz4, zstd, or on +#wal_init_zero = on # zero-fill new WAL files +#wal_recycle = on # recycle WAL files +#wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers + # (change requires restart) +#wal_writer_delay = 200ms # 1-10000 milliseconds +#wal_writer_flush_after = 1MB # measured in pages, 0 disables +#wal_skip_threshold = 2MB + +#commit_delay = 0 # range 0-100000, in microseconds +#commit_siblings = 5 # range 1-1000 + +# - Checkpoints - + +#checkpoint_timeout = 5min # range 30s-1d +#checkpoint_completion_target = 0.9 # checkpoint target duration, 0.0 - 1.0 +#checkpoint_flush_after = 256kB # measured in pages, 0 disables +#checkpoint_warning = 30s # 0 disables +max_wal_size = 1GB +min_wal_size = 80MB + +# - Prefetching during recovery - + +#recovery_prefetch = try # prefetch pages referenced in the WAL? +#wal_decode_buffer_size = 512kB # lookahead window used for prefetching + # (change requires restart) + +# - Archiving - + +#archive_mode = off # enables archiving; off, on, or always + # (change requires restart) +#archive_library = '' # library to use to archive a logfile segment + # (empty string indicates archive_command should + # be used) +#archive_command = '' # command to use to archive a logfile segment + # placeholders: %p = path of file to archive + # %f = file name only + # e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f' +#archive_timeout = 0 # force a logfile segment switch after this + # number of seconds; 0 disables + +# - Archive Recovery - + +# These are only used in recovery mode. + +#restore_command = '' # command to use to restore an archived logfile segment + # placeholders: %p = path of file to restore + # %f = file name only + # e.g. 'cp /mnt/server/archivedir/%f %p' +#archive_cleanup_command = '' # command to execute at every restartpoint +#recovery_end_command = '' # command to execute at completion of recovery + +# - Recovery Target - + +# Set these only when performing a targeted recovery. + +#recovery_target = '' # 'immediate' to end recovery as soon as a + # consistent state is reached + # (change requires restart) +#recovery_target_name = '' # the named restore point to which recovery will proceed + # (change requires restart) +#recovery_target_time = '' # the time stamp up to which recovery will proceed + # (change requires restart) +#recovery_target_xid = '' # the transaction ID up to which recovery will proceed + # (change requires restart) +#recovery_target_lsn = '' # the WAL LSN up to which recovery will proceed + # (change requires restart) +#recovery_target_inclusive = on # Specifies whether to stop: + # just after the specified recovery target (on) + # just before the recovery target (off) + # (change requires restart) +#recovery_target_timeline = 'latest' # 'current', 'latest', or timeline ID + # (change requires restart) +#recovery_target_action = 'pause' # 'pause', 'promote', 'shutdown' + # (change requires restart) + + +#------------------------------------------------------------------------------ +# REPLICATION +#------------------------------------------------------------------------------ + +# - Sending Servers - + +# Set these on the primary and on any standby that will send replication data. + +#max_wal_senders = 10 # max number of walsender processes + # (change requires restart) +#max_replication_slots = 10 # max number of replication slots + # (change requires restart) +#wal_keep_size = 0 # in megabytes; 0 disables +#max_slot_wal_keep_size = -1 # in megabytes; -1 disables +#wal_sender_timeout = 60s # in milliseconds; 0 disables +#track_commit_timestamp = off # collect timestamp of transaction commit + # (change requires restart) + +# - Primary Server - + +# These settings are ignored on a standby server. + +#synchronous_standby_names = '' # standby servers that provide sync rep + # method to choose sync standbys, number of sync standbys, + # and comma-separated list of application_name + # from standby(s); '*' = all +#vacuum_defer_cleanup_age = 0 # number of xacts by which cleanup is delayed + +# - Standby Servers - + +# These settings are ignored on a primary server. + +#primary_conninfo = '' # connection string to sending server +#primary_slot_name = '' # replication slot on sending server +#promote_trigger_file = '' # file name whose presence ends recovery +#hot_standby = on # "off" disallows queries during recovery + # (change requires restart) +#max_standby_archive_delay = 30s # max delay before canceling queries + # when reading WAL from archive; + # -1 allows indefinite delay +#max_standby_streaming_delay = 30s # max delay before canceling queries + # when reading streaming WAL; + # -1 allows indefinite delay +#wal_receiver_create_temp_slot = off # create temp slot if primary_slot_name + # is not set +#wal_receiver_status_interval = 10s # send replies at least this often + # 0 disables +#hot_standby_feedback = off # send info from standby to prevent + # query conflicts +#wal_receiver_timeout = 60s # time that receiver waits for + # communication from primary + # in milliseconds; 0 disables +#wal_retrieve_retry_interval = 5s # time to wait before retrying to + # retrieve WAL after a failed attempt +#recovery_min_apply_delay = 0 # minimum delay for applying changes during recovery + +# - Subscribers - + +# These settings are ignored on a publisher. + +#max_logical_replication_workers = 4 # taken from max_worker_processes + # (change requires restart) +#max_sync_workers_per_subscription = 2 # taken from max_logical_replication_workers + + +#------------------------------------------------------------------------------ +# QUERY TUNING +#------------------------------------------------------------------------------ + +# - Planner Method Configuration - + +#enable_async_append = on +#enable_bitmapscan = on +#enable_gathermerge = on +#enable_hashagg = on +#enable_hashjoin = on +#enable_incremental_sort = on +#enable_indexscan = on +#enable_indexonlyscan = on +#enable_material = on +#enable_memoize = on +#enable_mergejoin = on +#enable_nestloop = on +#enable_parallel_append = on +#enable_parallel_hash = on +#enable_partition_pruning = on +#enable_partitionwise_join = off +#enable_partitionwise_aggregate = off +#enable_seqscan = on +#enable_sort = on +#enable_tidscan = on + +# - Planner Cost Constants - + +#seq_page_cost = 1.0 # measured on an arbitrary scale +#random_page_cost = 4.0 # same scale as above +#cpu_tuple_cost = 0.01 # same scale as above +#cpu_index_tuple_cost = 0.005 # same scale as above +#cpu_operator_cost = 0.0025 # same scale as above +#parallel_setup_cost = 1000.0 # same scale as above +#parallel_tuple_cost = 0.1 # same scale as above +#min_parallel_table_scan_size = 8MB +#min_parallel_index_scan_size = 512kB +#effective_cache_size = 4GB + +#jit_above_cost = 100000 # perform JIT compilation if available + # and query more expensive than this; + # -1 disables +#jit_inline_above_cost = 500000 # inline small functions if query is + # more expensive than this; -1 disables +#jit_optimize_above_cost = 500000 # use expensive JIT optimizations if + # query is more expensive than this; + # -1 disables + +# - Genetic Query Optimizer - + +#geqo = on +#geqo_threshold = 12 +#geqo_effort = 5 # range 1-10 +#geqo_pool_size = 0 # selects default based on effort +#geqo_generations = 0 # selects default based on effort +#geqo_selection_bias = 2.0 # range 1.5-2.0 +#geqo_seed = 0.0 # range 0.0-1.0 + +# - Other Planner Options - + +#default_statistics_target = 100 # range 1-10000 +#constraint_exclusion = partition # on, off, or partition +#cursor_tuple_fraction = 0.1 # range 0.0-1.0 +#from_collapse_limit = 8 +#jit = on # allow JIT compilation +#join_collapse_limit = 8 # 1 disables collapsing of explicit + # JOIN clauses +#plan_cache_mode = auto # auto, force_generic_plan or + # force_custom_plan +#recursive_worktable_factor = 10.0 # range 0.001-1000000 + + +#------------------------------------------------------------------------------ +# REPORTING AND LOGGING +#------------------------------------------------------------------------------ + +# - Where to Log - + +#log_destination = 'stderr' # Valid values are combinations of + # stderr, csvlog, jsonlog, syslog, and + # eventlog, depending on platform. + # csvlog and jsonlog require + # logging_collector to be on. + +# This is used when logging to stderr: +#logging_collector = off # Enable capturing of stderr, jsonlog, + # and csvlog into log files. Required + # to be on for csvlogs and jsonlogs. + # (change requires restart) + +# These are only used if logging_collector is on: +#log_directory = 'log' # directory where log files are written, + # can be absolute or relative to PGDATA +#log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # log file name pattern, + # can include strftime() escapes +#log_file_mode = 0600 # creation mode for log files, + # begin with 0 to use octal notation +#log_rotation_age = 1d # Automatic rotation of logfiles will + # happen after that time. 0 disables. +#log_rotation_size = 10MB # Automatic rotation of logfiles will + # happen after that much log output. + # 0 disables. +#log_truncate_on_rotation = off # If on, an existing log file with the + # same name as the new log file will be + # truncated rather than appended to. + # But such truncation only occurs on + # time-driven rotation, not on restarts + # or size-driven rotation. Default is + # off, meaning append to existing files + # in all cases. + +# These are relevant when logging to syslog: +#syslog_facility = 'LOCAL0' +#syslog_ident = 'postgres' +#syslog_sequence_numbers = on +#syslog_split_messages = on + +# This is only relevant when logging to eventlog (Windows): +# (change requires restart) +#event_source = 'PostgreSQL' + +# - When to Log - + +#log_min_messages = warning # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # log + # fatal + # panic + +#log_min_error_statement = error # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # log + # fatal + # panic (effectively off) + +#log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements + # and their durations, > 0 logs only + # statements running at least this number + # of milliseconds + +#log_min_duration_sample = -1 # -1 is disabled, 0 logs a sample of statements + # and their durations, > 0 logs only a sample of + # statements running at least this number + # of milliseconds; + # sample fraction is determined by log_statement_sample_rate + +#log_statement_sample_rate = 1.0 # fraction of logged statements exceeding + # log_min_duration_sample to be logged; + # 1.0 logs all such statements, 0.0 never logs + + +#log_transaction_sample_rate = 0.0 # fraction of transactions whose statements + # are logged regardless of their duration; 1.0 logs all + # statements from all transactions, 0.0 never logs + +#log_startup_progress_interval = 10s # Time between progress updates for + # long-running startup operations. + # 0 disables the feature, > 0 indicates + # the interval in milliseconds. + +# - What to Log - + +#debug_print_parse = off +#debug_print_rewritten = off +#debug_print_plan = off +#debug_pretty_print = on +#log_autovacuum_min_duration = 10min # log autovacuum activity; + # -1 disables, 0 logs all actions and + # their durations, > 0 logs only + # actions running at least this number + # of milliseconds. +#log_checkpoints = on +#log_connections = off +#log_disconnections = off +#log_duration = off +#log_error_verbosity = default # terse, default, or verbose messages +#log_hostname = off +#log_line_prefix = '%m [%p] ' # special values: + # %a = application name + # %u = user name + # %d = database name + # %r = remote host and port + # %h = remote host + # %b = backend type + # %p = process ID + # %P = process ID of parallel group leader + # %t = timestamp without milliseconds + # %m = timestamp with milliseconds + # %n = timestamp with milliseconds (as a Unix epoch) + # %Q = query ID (0 if none or not computed) + # %i = command tag + # %e = SQL state + # %c = session ID + # %l = session line number + # %s = session start timestamp + # %v = virtual transaction ID + # %x = transaction ID (0 if none) + # %q = stop here in non-session + # processes + # %% = '%' + # e.g. '<%u%%%d> ' +#log_lock_waits = off # log lock waits >= deadlock_timeout +#log_recovery_conflict_waits = off # log standby recovery conflict waits + # >= deadlock_timeout +#log_parameter_max_length = -1 # when logging statements, limit logged + # bind-parameter values to N bytes; + # -1 means print in full, 0 disables +#log_parameter_max_length_on_error = 0 # when logging an error, limit logged + # bind-parameter values to N bytes; + # -1 means print in full, 0 disables +#log_statement = 'none' # none, ddl, mod, all +#log_replication_commands = off +#log_temp_files = -1 # log temporary files equal or larger + # than the specified size in kilobytes; + # -1 disables, 0 logs all temp files +log_timezone = 'UTC' + + +#------------------------------------------------------------------------------ +# PROCESS TITLE +#------------------------------------------------------------------------------ + +#cluster_name = '' # added to process titles if nonempty + # (change requires restart) +#update_process_title = on + + +#------------------------------------------------------------------------------ +# STATISTICS +#------------------------------------------------------------------------------ + +# - Cumulative Query and Index Statistics - + +#track_activities = on +#track_activity_query_size = 1024 # (change requires restart) +#track_counts = on +#track_io_timing = off +#track_wal_io_timing = off +#track_functions = none # none, pl, all +#stats_fetch_consistency = cache + + +# - Monitoring - + +#compute_query_id = auto +#log_statement_stats = off +#log_parser_stats = off +#log_planner_stats = off +#log_executor_stats = off + + +#------------------------------------------------------------------------------ +# AUTOVACUUM +#------------------------------------------------------------------------------ + +#autovacuum = on # Enable autovacuum subprocess? 'on' + # requires track_counts to also be on. +#autovacuum_max_workers = 3 # max number of autovacuum subprocesses + # (change requires restart) +#autovacuum_naptime = 1min # time between autovacuum runs +#autovacuum_vacuum_threshold = 50 # min number of row updates before + # vacuum +#autovacuum_vacuum_insert_threshold = 1000 # min number of row inserts + # before vacuum; -1 disables insert + # vacuums +#autovacuum_analyze_threshold = 50 # min number of row updates before + # analyze +#autovacuum_vacuum_scale_factor = 0.2 # fraction of table size before vacuum +#autovacuum_vacuum_insert_scale_factor = 0.2 # fraction of inserts over table + # size before insert vacuum +#autovacuum_analyze_scale_factor = 0.1 # fraction of table size before analyze +#autovacuum_freeze_max_age = 200000000 # maximum XID age before forced vacuum + # (change requires restart) +#autovacuum_multixact_freeze_max_age = 400000000 # maximum multixact age + # before forced vacuum + # (change requires restart) +#autovacuum_vacuum_cost_delay = 2ms # default vacuum cost delay for + # autovacuum, in milliseconds; + # -1 means use vacuum_cost_delay +#autovacuum_vacuum_cost_limit = -1 # default vacuum cost limit for + # autovacuum, -1 means use + # vacuum_cost_limit + + +#------------------------------------------------------------------------------ +# CLIENT CONNECTION DEFAULTS +#------------------------------------------------------------------------------ + +# - Statement Behavior - + +#client_min_messages = notice # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # log + # notice + # warning + # error +#search_path = '"$user", public' # schema names +#row_security = on +#default_table_access_method = 'heap' +#default_tablespace = '' # a tablespace name, '' uses the default +#default_toast_compression = 'pglz' # 'pglz' or 'lz4' +#temp_tablespaces = '' # a list of tablespace names, '' uses + # only default tablespace +#check_function_bodies = on +#default_transaction_isolation = 'read committed' +#default_transaction_read_only = off +#default_transaction_deferrable = off +#session_replication_role = 'origin' +#statement_timeout = 0 # in milliseconds, 0 is disabled +#lock_timeout = 0 # in milliseconds, 0 is disabled +#idle_in_transaction_session_timeout = 0 # in milliseconds, 0 is disabled +#idle_session_timeout = 0 # in milliseconds, 0 is disabled +#vacuum_freeze_table_age = 150000000 +#vacuum_freeze_min_age = 50000000 +#vacuum_failsafe_age = 1600000000 +#vacuum_multixact_freeze_table_age = 150000000 +#vacuum_multixact_freeze_min_age = 5000000 +#vacuum_multixact_failsafe_age = 1600000000 +#bytea_output = 'hex' # hex, escape +#xmlbinary = 'base64' +#xmloption = 'content' +#gin_pending_list_limit = 4MB + +# - Locale and Formatting - + +datestyle = 'iso, mdy' +#intervalstyle = 'postgres' +timezone = 'UTC' +#timezone_abbreviations = 'Default' # Select the set of available time zone + # abbreviations. Currently, there are + # Default + # Australia (historical usage) + # India + # You can create your own file in + # share/timezonesets/. +#extra_float_digits = 1 # min -15, max 3; any value >0 actually + # selects precise output mode +#client_encoding = sql_ascii # actually, defaults to database + # encoding + +# These settings are initialized by initdb, but they can be changed. +lc_messages = 'en_US.utf8' # locale for system error message + # strings +lc_monetary = 'en_US.utf8' # locale for monetary formatting +lc_numeric = 'en_US.utf8' # locale for number formatting +lc_time = 'en_US.utf8' # locale for time formatting + +# default configuration for text search +default_text_search_config = 'pg_catalog.english' + +# - Shared Library Preloading - + +#local_preload_libraries = '' +#session_preload_libraries = '' +shared_preload_libraries = 'pg_stat_statements' # (change requires restart) +#jit_provider = 'llvmjit' # JIT library to use + +# - Other Defaults - + +#dynamic_library_path = '$libdir' +#gin_fuzzy_search_limit = 0 + + +#------------------------------------------------------------------------------ +# LOCK MANAGEMENT +#------------------------------------------------------------------------------ + +#deadlock_timeout = 1s +#max_locks_per_transaction = 64 # min 10 + # (change requires restart) +#max_pred_locks_per_transaction = 64 # min 10 + # (change requires restart) +#max_pred_locks_per_relation = -2 # negative values mean + # (max_pred_locks_per_transaction + # / -max_pred_locks_per_relation) - 1 +#max_pred_locks_per_page = 2 # min 0 + + +#------------------------------------------------------------------------------ +# VERSION AND PLATFORM COMPATIBILITY +#------------------------------------------------------------------------------ + +# - Previous PostgreSQL Versions - + +#array_nulls = on +#backslash_quote = safe_encoding # on, off, or safe_encoding +#escape_string_warning = on +#lo_compat_privileges = off +#quote_all_identifiers = off +#standard_conforming_strings = on +#synchronize_seqscans = on + +# - Other Platforms and Clients - + +#transform_null_equals = off + + +#------------------------------------------------------------------------------ +# ERROR HANDLING +#------------------------------------------------------------------------------ + +#exit_on_error = off # terminate session on any error? +#restart_after_crash = on # reinitialize after backend crash? +#data_sync_retry = off # retry or panic on failure to fsync + # data? + # (change requires restart) +#recovery_init_sync_method = fsync # fsync, syncfs (Linux 5.8+) + + +#------------------------------------------------------------------------------ +# CONFIG FILE INCLUDES +#------------------------------------------------------------------------------ + +# These options allow settings to be loaded from files other than the +# default postgresql.conf. Note that these are directives, not variable +# assignments, so they can usefully be given more than once. + +#include_dir = '...' # include files ending in '.conf' from + # a directory, e.g., 'conf.d' +#include_if_exists = '...' # include file only if it exists +#include = '...' # include file + + +#------------------------------------------------------------------------------ +# CUSTOMIZED OPTIONS +#------------------------------------------------------------------------------ + +# Add settings for extensions here diff --git a/docker/run_node_postgres.sh b/docker/run_node_postgres.sh new file mode 100644 index 0000000..b04c4d3 --- /dev/null +++ b/docker/run_node_postgres.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +## Install the `dig` command +apk add --update bind-tools + +peer_IP=$(dig +short nwaku_sqlite) + +exec /usr/bin/wakunode\ + --nodekey=1d714a1fada214dead6dc9c7274585eca0ff292451866e7d6d677dc818e8ccd2\ + --staticnode=/ip4/${peer_IP}/tcp/30304/p2p/16Uiu2HAkxj3WzLiqBximSaHc8wV9Co87GyRGRYLVGsHZrzi3TL5W\ + --relay=true\ + --topic=/waku/2/default-waku/proto\ + --topic=/waku/2/dev-waku/proto\ + --filter=true\ + --lightpush=true\ + --rpc-admin=true\ + --keep-alive=true\ + --max-connections=150\ + --log-level=DEBUG\ + --dns-discovery=true\ + --rpc-port=8545\ + --rpc-address=0.0.0.0\ + --tcp-port=30303\ + --metrics-server=True\ + --metrics-server-port=8003\ + --metrics-server-address=0.0.0.0\ + --store-message-db-url="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/postgres"\ + --store=true\ + --store-message-retention-policy=time:864000 diff --git a/docker/run_node_sqlite.sh b/docker/run_node_sqlite.sh new file mode 100644 index 0000000..2222ffa --- /dev/null +++ b/docker/run_node_sqlite.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +## Install the `dig` command +apk add --update bind-tools + +peer_IP=$(dig +short nwaku_postgres) + +exec /usr/bin/wakunode\ + --nodekey=2d714a1fada214dead6dc9c7274585eca0ff292451866e7d6d677dc818e8ccd2\ + --staticnode=/ip4/${peer_IP}/tcp/30303/p2p/16Uiu2HAmJyLCRhiErTRFcW5GKPrpoMjGbbWdFMx4GCUnnhmxeYhd\ + --relay=true\ + --topic=/waku/2/default-waku/proto\ + --topic=/waku/2/dev-waku/proto\ + --filter=true\ + --lightpush=true\ + --rpc-admin=true\ + --keep-alive=true\ + --max-connections=150\ + --log-level=DEBUG\ + --dns-discovery=true\ + --rpc-port=8546\ + --rpc-address=0.0.0.0\ + --tcp-port=30304\ + --metrics-server=True\ + --metrics-server-port=8004\ + --metrics-server-address=0.0.0.0\ + --store-message-db-url="sqlite:///data/store.sqlite3"\ + --store=true\ + --store-message-retention-policy=time:864000 diff --git a/docker/store_query_producer.sh b/docker/store_query_producer.sh new file mode 100644 index 0000000..325ad98 --- /dev/null +++ b/docker/store_query_producer.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +## Install the `dig` command +apk --no-cache add bind-tools + +apk --no-cache add curl + +## Install coreutils so that the `date` command behaves as the GNU one +apk --no-cache add coreutils + +## The port numbers and host names are defined in the 'docker-compose.yml' file +nwaku_postgres_IP=$(dig +short nwaku_postgres) +nwaku_sqlite_IP=$(dig +short nwaku_sqlite) +target_postgres="http://${nwaku_postgres_IP}:8545" +target_sqlite="http://${nwaku_sqlite_IP}:8546" + +echo "This is publisher: ${target_postgres}; ${target_sqlite}" + +## Wait a few seconds until the `nwaku` nodes started their rpc services +sleep 20 + +while true +do + ## Send a 'get_waku_v2_store_v1_messages' req to the ""postgres"" node + curl -d '{"jsonrpc":"2.0","id":"id","method":"get_waku_v2_store_v1_messages"}' --header "Content-Type: application/json" ${target_postgres} + + ## Send a 'get_waku_v2_store_v1_messages' req to the ""sqlite"" node + curl -d '{"jsonrpc":"2.0","id":"id","method":"get_waku_v2_store_v1_messages"}' --header "Content-Type: application/json" ${target_sqlite} + + sleep 5 +done diff --git a/postgres-docker-compose.yml b/postgres-docker-compose.yml deleted file mode 100644 index 99f93e6..0000000 --- a/postgres-docker-compose.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: "3.8" - -services: - db: - image: postgres:9.6-alpine - restart: always - environment: - POSTGRES_PASSWORD: test123 - ports: - - "5432:5432"