From d79f78cc55294e503ae3e5fc45b025e6e8e9aec6 Mon Sep 17 00:00:00 2001 From: gfc079 Date: Tue, 6 Feb 2024 11:24:10 -0500 Subject: [PATCH 1/2] Added AWS S3 Query runner --- client/app/assets/images/db-logos/s3.png | Bin 0 -> 3133 bytes redash/query_runner/s3.py | 114 +++++++++++++++++++++++ redash/settings/__init__.py | 1 + 3 files changed, 115 insertions(+) create mode 100644 client/app/assets/images/db-logos/s3.png create mode 100644 redash/query_runner/s3.py diff --git a/client/app/assets/images/db-logos/s3.png b/client/app/assets/images/db-logos/s3.png new file mode 100644 index 0000000000000000000000000000000000000000..e87a07bc52455f4d3ef518c2456f2056d42a687b GIT binary patch literal 3133 zcmZ{nc`(}x*T<=Awn0ihR|9{1eaLa+VzsA zwX{JkQA={$QnjzG+UuehzsEoCyz@LW&&)aJb7s!WneRVm=6llZY%F-rUpmjm#>RuV z0dqLz+-c5&I8G~Fb9?<1({|QKbKth!tbq9BhA}k zSMX|*z~lsg2?M6T0zL15+G^nYH-I?`z;uAN7T|3MQ1u5;oIhiA4Y-2>@Hl{g2U?pQ zE(xW`3;97V#VAUU^pqY-fUwG9)rRWUq5|_?w^neGYuPEAmE_@!N3_Vm(wkQ9M2%t36SIh>7V0L@Z~5z&TC3ce3(+jWFV0J$Box{Z4rS&a;%_1D!qd z@cMsXTB>F`sFTWL;$RcEkF!@3DHwacc`lzpNTfMg!!LE2R?)!>bKg8%MpHzl4NdD+ zT@Q%6_t}Dog6ahs6k1J%3ru(y@IJh(i7j()#ZoRmM{8NE*_aI~Wr$&y<}2-bzy}&Z z_ew_-3lHx=S6s3;W!wPW!KG#P;%V5b{!;N&6y^Sv%ywh%l+Qs+mA4w@qe**&sRy4> zPqR*n{t~Z9)U(3m?%cxXHQ1L&x05UJEA}W|^J5Y1uW3uq@x%&(TFB4jZZc+x1&Tji z(VECI$Qu1d^*ASAVtR05wXO-h$oS+Z?{x_089L&3D7ii!{$qiC&Op#9jFhfIFxQX8 z;$pwF4jw1+gO#SbUP(wulzO-wSNEu%E3al@4-UUArMz!Jv(Zs2Q+Elg{DtINC_p&C%fC*fe{+GLkP0)7~2h6SztM8l`!x^ZP^86i4p;WHp!cEiwNA+hhq!X%t;C62ypn4xsTvjhlJ2(yv^W!7;vkvj1>rV_ zvr3J4@{Z&dSc&h9G+(Fs+|<<6htx7-oFe43rM^KMpZuEqxbuCjW~9sCXE{BGo(~*v zGs2x87dShkO@JDxW_ILqN|PdRRwc@Ka_ z=GG2w+msy#_^;=5&1}{36OVSVl%oxFuO2KbNJ3{NtLtb^|AW(7=79r}j1D)DmpV(| z72!KGcgPKTxOH?3M^E(2F?Fe3k^u~rYZ47^$PC4urIBA>>^m4KEwMV@$M|WH@aAb{ z4n5Dn+=3=@z0W`dArrZG{a|hp6S?;{z}$>J*gg^d_^^4s~6DXE!yyL3M0eSImE4@^AKa*E+Ge=t}KgOo94AR&af?EoGT^OM38>P7irVR!eo7SSQN*3=VDr|c>EQvM9>X4k9r7|?XXF(bFx|uFBiq>m< z`2GStH6}G2@>X6LYytUYh8C1y>B{|y=St-Ti;t!DL6aI&qonUXw}>|r-^K<{sBCA{ zlRtS-bI0kcyH{vxvpFU{KNzn1+^)g>v=ygIe~??dbBvjxyPr)1_!@3wUQp(0mmc-@ zJ`rJwjVZPi+XyynK8X-aj#%?nUD@am=O4bS*talFwvYWFF7&>SgoaM`U)*>iR%Y6t z(CrLvg+ES7DU1gBE9RX9n>p2c@e zPmHBe4vVVuELI6J%UBa$=x63sC@bsj#fQp{CV;5Fb4J@YR}^b7A7{xrrv)`G;JF+m zNY<`6`Lji0kUiduHv@%{^&9*x#d6YRBi^&YjnYi_oA0q^EPRlu>rN^oP*GW!MqtIs zf)C|Bu|s7rAbBe9U7zxnf(}<^V;_Dr0XL@uLcsdDTvb!X(VWd}!orhp0Q{$G80 zKBhHQ)QHRWgw^eP1}+n-?7sY{*S0d!c_E(%4;V7LZTTaTl+)9pVhA9k=4x8B zG}Ui}@~70Vh1GA35agjc`g}st=o6m~@h_3-PrXoI&Z+SPEL&1kd|%5YQ_3?s>|GeT zvU|4?*s>KJhV=QDPySdeGv1|4D2@K9gL0h9j_Xk1Y-^RbNch0eO<(UWA`VLj%FM22nu-iK` zvJ~^}i0TE#kk%IsKODUkEpGN-QKJd}ydea=jIAHCaS6KtoYP3v1UD#JCU8c-fC|b?`qrK=LRh*1aOwj$!yw`hj zezaRy?n+~zW#<<3yQK_EgF>v|iImN?a~^(Zpjw{*Z3}$1ZjfWsjbx5kN9_grC8a+jhQ1Vql3`UpQVYCD z?YZ^Wz?f}a*;}2~v=<(!TRzYEi>Zk}W6f9Lc*EXkJ>?9?M-!Q=TS1st;Lzr*=BZ0l z1YP#}Jzw`{ELhU03Cv}Odbfb4|008n`t&NC7V=l=Pb{H+1e5u^KWwb}rrdmX*hFxr z_MC3KQucAqjO*W625+O)@o=llblrI2ySCda8Z-}L(nLh^UtXE9c>Q+RM)5M!ZGxJ~AR0pMkZyh| z`Q}Y?2@Rb&SbTerTDR9`rFEWQm{wheMBpaQOFsE8+xmYd_K)ae&W}rx4eIaM3{U?7 NHiWqitO4rz=-)^d{s{m8 literal 0 HcmV?d00001 diff --git a/redash/query_runner/s3.py b/redash/query_runner/s3.py new file mode 100644 index 0000000000..3cd055f80f --- /dev/null +++ b/redash/query_runner/s3.py @@ -0,0 +1,114 @@ +import boto3 +import pandas as pd +from redash.query_runner import BaseQueryRunner, register +from redash.query_runner import TYPE_STRING, TYPE_INTEGER, TYPE_BOOLEAN, TYPE_FLOAT, TYPE_DATE, TYPE_DATETIME +from redash.utils import json_dumps, json_loads +import logging + +TYPES_MAP = { + "bool": TYPE_BOOLEAN, + "datetime64[ns]": TYPE_DATETIME, + "datetime64[s]": TYPE_DATETIME, + "float64": TYPE_FLOAT, + "int64": TYPE_INTEGER, + "object": TYPE_STRING +} + +logger = logging.getLogger(__name__) + +class S3(BaseQueryRunner): + @classmethod + def name(cls): + return "Amazon S3" + @classmethod + def configuration_schema(cls): + return { + "type": "object", + "properties": { + "region": {"type": "string", "title": "AWS Region"}, + "bucket_name": {"type": "string", "title": "Bucket Name"}, + "object_key": {"type": "string", "title": "Object Key"} + }, + "required": ["region", "bucket_name", "object_key"], + "order": ["region", "bucket_name", "object_key"], + } + def test_connection(self): + region = self.configuration["region"] + bucket_name = self.configuration["bucket_name"] + object_key = self.configuration["object_key"] + + # Set S3 client using Boto3 + s3_client = boto3.client("s3") + + query = "SELECT * from S3Object" + # As of now we are required to pass in the object key so we are configuring the data source to a particular S3 object temporarily + resp = s3_client.select_object_content( + Bucket=bucket_name, + Key= object_key, # We need the CSV file (Object Key) + ExpressionType='SQL', + Expression=query, + InputSerialization = {'CSV': {"FileHeaderInfo": "Use"}, 'CompressionType': 'NONE'}, + OutputSerialization = {'JSON': {}}, + ) + + # Need to first deploy this to see how response data schema is before we can parse it into rows/columns + for event in resp['Payload']: + if 'Records' in event: + records = event['Records']['Payload'] + logger.info("Records: %s", records) + + def run_query(self, query, user): + region = self.configuration["region"] + bucket_name = self.configuration["bucket_name"] + object_key = self.configuration["object_key"] + + # Set S3 client using Boto3 + s3_client = boto3.client("s3") + + # As of now we are required to pass in the object key so we are configuring the data source to a particular S3 object temporarily + resp = s3_client.select_object_content( + Bucket=bucket_name, + Key= object_key, # We need the CSV file (Object Key) + ExpressionType='SQL', + Expression=query, + InputSerialization = {'CSV': {"FileHeaderInfo": "Use"}, 'CompressionType': 'NONE'}, + OutputSerialization = {'JSON': {}}, + ) + + # Need to first deploy this to see how response data schema is before we can parse it into rows/columns + json_result = "" + for event in resp['Payload']: + if 'Records' in event: + json_result = event['Records']['Payload'] + logger.info("Records: %s", json_result) + + json_result = json_result.decode('utf8') + json_result = json_result.replace('\n', '') + json_result = json_result.replace('\\r', '') + json_result = json_result.replace('}{', '},{') + json_result = "[" + json_result + "]" + logger.info("JSON: %s", json_result) + dict_result = json_loads(json_result) + logger.info("DictResult: %s", dict_result) + df = pd.DataFrame(dict_result) + logger.info("DataFrame: %s", df.to_string()) + columns = [] + rows = df.to_dict('records') + + for col in df.columns: + columns.append( + { + "name": col, + "friendly_name": col, + "type": TYPES_MAP[str(df[col].dtype)] + } + ) + + # Returning the query results in Redash format + data = {"columns": columns, "rows": rows} + error = None + json_data = json_dumps(data) + return json_data, error + +# Registering custom S3 query runner +register(S3) \ No newline at end of file diff --git a/redash/settings/__init__.py b/redash/settings/__init__.py index b7d30c693d..7d44a3e80b 100644 --- a/redash/settings/__init__.py +++ b/redash/settings/__init__.py @@ -280,6 +280,7 @@ def email_server_is_configured(): "redash.query_runner.google_spreadsheets", "redash.query_runner.graphite", "redash.query_runner.mongodb", + "redash.query_runner.s3", "redash.query_runner.couchbase", "redash.query_runner.mysql", "redash.query_runner.pg", From 33732d02300bb383175b62557eedee1880c54710 Mon Sep 17 00:00:00 2001 From: gfc079 Date: Thu, 8 Feb 2024 21:41:34 -0500 Subject: [PATCH 2/2] Added new AWS Dynamodb query runner/datasource plugin --- .../app/assets/images/db-logos/dynamodb.png | Bin 0 -> 12572 bytes redash/query_runner/dynamodb.py | 182 ++++++++++++++++++ redash/settings/__init__.py | 3 +- 3 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 client/app/assets/images/db-logos/dynamodb.png create mode 100644 redash/query_runner/dynamodb.py diff --git a/client/app/assets/images/db-logos/dynamodb.png b/client/app/assets/images/db-logos/dynamodb.png new file mode 100644 index 0000000000000000000000000000000000000000..22d9e6430b3df508d83b45168b79dfc4de1c12e5 GIT binary patch literal 12572 zcmch-Wl$Ya)Fs;Ya&dQex8MYKcMWdA5+p!?K#*|p5Zv9}-RY_R`-duwlnEL*ES$kO1{AV74K_dYv5(edB z0jqCC?=F^_ew*>On>0{8?nI%pI`O32wofQSTi=S^(P$^~9ayZ6wSIhllkqk_9Xan( z8fzzs7nJ}ZFF3VOv&A;Ptb1iTphrV@CtYu)UYU=`x}!lh$90teU&Oj=zHo(>{&Utq z=)<&#$>m|Wf$I?2r{|tnZIzgU+D^a5B8Kt29++jzFV8TJAzH7!)dF%@Y{xtYM%SJF zQ(U8Qjj67+U;^*3vxPDsdCsXD#fY3^_Inx-^Lx50hM(H*8?6YfE~2Z)%>HXm(fJnuJ{`?O}EPO|Bdz>J|oU7=p}UHMy@<{X6I zaSo{1S6V3N$2SGH+ z^zi9Rib1+r=2P{d5znVb%J2MA?*bbi(~3cokDTLiBj#FGGjJE|dtej5Cy5oCLdYxR zudlhP+KVHj)R-0v?sL5XXSp}km+;Kw{N)TB!%p*G=;|v*;;z3u#wQ5a4_d5;LQwra z@E3c({;KZ)HWgR~HU~i|$25LPF#=}^JY`mT_ELgljA~Cfnl&4aR>SnzJ7zBxHzd| zZ?x8~@BI^umftqPpEmw!1>&SIM0557jM23d>b5jvLV+fokPwOr46?bWyPQ z+o6Z2l5G-F6OZ&aDWYixHud70?r8D1i>%LKS(`RT5Mv%O;(40yKjsyRga!J)C)t`_ zL&KYj64_#SqB=(}*dt!Qv7q9yw@a=eJd{dd;<&}KaDKkN&bYup5g=&yH*>n*8NLq@!u7)+`6^AWwoV6Id2Q%(o|QPwj^&izs0e!fU4BO$ z&;mWPJ%-r}{|>`kwsqV7_?+lz1FY6p%l$DgpM$k)?HAwPYDlXC`^}Q|T`D2-{ z&g5=TKY|}zh($^&_;l&p!A(D23>%6pgO|wI5hi!f0@lsjFWgQh>9lpiTVVuPqB&S< z>M>mO2%Csh0tb!^I*S z1-ChS&a+6JHo85s#aqC<$2eza9v!C+)Vleak6QU$|b^L)<3h|@5fP|d59X~dfJvHbbuR{=U{FVXU$f- zr-=dztQN*gM|e0T`G2k2|0lZsUr>8B@RW0QDCOKh?cL`MhRgH8X7X;dqdaeR8{HI} zTb_=Tf6^F@m<50q81blgvUmI-*ImOMn`=s!N7gA*m> zUKe1~-AK-7Je+Oe3o9NJV{2n`hVpoQsa~PmW9xX);|m_cO8psg5mS=Chm`)XIBJs9 z^X}V>Gv$i`E?{#_HQYjfXdwgd*G~>2k*7s?(rOA{$-%Hrs4OnZU#7!yd6uT3XH zf0B#%C)NZS*X? zz3=Mx0aYo$GbrI)cv@7{?&v*US=-e+mQ&jyI^B$_{EcA!lI1z-?`dh>#-tW9(^R8I|b89_eDb60~y(QvdY7 zyRK#fg%V;tV-5?%fQA{zP3*KO)7EYjAQ7Bkh|4BLL#h@U^~}2b)akd68cqrTlgeA#ihZ}VWa1ZsC9Pg*w+rzNP+q7aC*-hf{IGq5dO`8*EVbCjKuYt- zV>J5fmcRY1ChRNzO^THu(hUTSgt0iGqws&xt+^JQ<2`gOtCjZ1soOmRJxf#meHXOG zSz+RYBr+zAF`|lEop1|I!wAy{1BTI9KMBbNkhrQVLq~Smsq8m`M%1q!2kIdmY*F(U&|Rr=mIIrP_mP8|QO zVEs>w_?OD&IP~?6Mxyh-_3EPT*jQgSDPu_Y5ukNB3%DM7nRxqdC2QQ2Pb{4q8Mx+o zrdO#ptDZ34PZw|>Q&yYfVEF8e@UV49o+mbAat@zLmVKMP&u@IoIBxjZ$dDk1E`1cwxrfvcV>AI^39qk64atl^&(ahZexqI=6i_9{DcS3o^oA|K^UfN(%t7OlW>mr`>lpECgXte{l)X{?Td#lfV^s5|FaMI>RW{nM^`vOpR*xP!#;HRf z{E~YupuJ;h0(yo-XNUPSbXhq`?{BRf;aGhu-3%*p^Oz4Wj_R6F^L2}nl|%Tt!;pNX zy4Y{bA+bet{Lixe<)7;ZmW^ruD_4$qUw0n!D^$dvw1GNm#_trt(44IwMjENTjVXoZ zJTh()sD2^bsrzzn@ta@8PN~c;8cBr-O@>cx#bor?vHzlBX1r$<^9juqbwRs_&+Ax& z+KXfOT{4*6L%itck_zp6>Ln2HZLV7{Zw`0057$#K&=1RFY)as)&PMPnp_poAdGa`e z_n4#kz7O=+^BpdB12b%r6GyS-yb`wm_R>?C#O(EDv#v%)U{2bWuAO^2Dc@FVdY&5g;K2ZV+G1%|2 z-OS06c%R6`a7TBIs2m2z17SqzaFlRG&KQ-Tn6AOf=7#o{I8EndSP>oW+93qe{Q zJKXPwXVFx?XQRoUgY{V}YAwaI$)FfNO$^#}WJD-2_9Va(ezh1IGQLuT6YRDu;1wKP{J1_{ zWGagAs-U$?{b2os-Nw@=a}^3wBaNiw-=1ORKcLziFIQqQzuzVrf|i=0>jSj=(cUtP zXK-x+$}O8!kwk9|Ty(*ctp0~4JvDqrFQ5-bec6n|;s{}4MEro`=jYgAypqpr=NwQf zc{c9BdvU>DHFDqR$Slf@x?XvjPVhmG801!`qH3{DdB6$m-Q}0~T#>O=)H(VZm#R>{ zWS*sLq`baQ3ue~Fgxkh~xMYB%jolri#lBDgk+9#rz>%;I)bGy#O0J6$X*`}!2lP|O zIyjBMtWu#1i=Y|naD=4%Q(@jdiy|!-#j_YBCCXYq@@1MH5cES^b;?uVIM zwgdy!Q)#lJ@dD5(g$XOIGEw#F0z)4x;6C8Hww3=e49O8__KupGk*b!I?Lz(J!m)J; zN26DkI^q|P>CgVijN7L|Av&NTY(>V-dsx!K`r+NyX?0uD7k-aBTYYJJBAxb(d6}Kn zfknu`!CUgrY+Gs3k_20+1(h&IFM3XGc@3LQb6Vj8q_kuLu2d7NZ(uY099kx}pJ%gA z72NF-@bG=Ts^pTKLIO=U{&QUpCqJLY63Y}xd8vKMN>(B$1Ig|!WO3)ba}npC@e>;dqXAK8f^<6)CZkbMPKUIBI z=4NgG5VG2Yj)4!-a7iX){}4LP`gd-_Wr0IBYPHU?FIaqxF576VoB@>L?=Bqx6`ki{ z`2LvzYWKsLXQ)gS*n_6<$uQ$e@X5ENE+;6y+T*R8wceycdro_1ka5h54niGGla87v zqh%+vJBKW_5uyLgpZ+Jx(SPM!|BJt&g7ar?HGW(nrELF6Xy?hLvA#EsbhzC=3l z)pvS7d);$H_^V(AcvPDOG}d}jTMn=2g?t^3-I;4hud%Q-T%>Q3IbN#)`xhKJEN6aG zJnFIX72Cm5h%bsc7O+`Az2&39H47kz#BZL3{2@~XB)gOCaaX_u*FmIVqf@#Q>+Z}a z?#6TSIQo1cc@Kj_3neNm@XKw5!TCDK68i}XNcj;1@BS{HmUgy%ZOoj3=}H58rZ4)K zxC4X|bGCTRv&a&DT6&|3bSQK+m)<#i+gu24<0uqfpa&M; zJQHf{dEn!JkOVmJLRu$cDRquzYKJeGYRca^+|8Nc;aA}Ze9`@lXIg)zp33C2PNwEG zT3jUk^{&(4&#N-EK`((2fg%$}ZA=l=3w-?HtPX}=9eXTqIyV~wbfNO}6^ zl)~{&X-`75=Aw5=zoq?+xDUIVQTz5VMq>o`m&+CRU@A}3TDv1#6@ch)3Y#EQLC?|> z>#$JoLzoSPea6(l-ka`-IsQh;kMumQ)djj>iHv}up`etdM%yF4>;!D5LA}-@d@9aO z;dimwEUihViD4HpGwOj}(Q0yD#59!FJy8^r=QX+h@eco-vi8%cbk_U#j8BQ@r@8c4 zz3SXF%g>m!Q%})P|8y6GRC0S%DFRUb`d);b^F`w4?Jx+9Kc)4LO9xF_srTO_FzRn! zuChJfpWaJ5%*TZrj7OY|{R(1ZrmHWH6LHZ550iB*-t}Q^dlfMLO0Be4ufa(fjauE` zh^5+lW6H@j9Mczl^1@y)v(Uk}iJD-3|wEtKf>&-8wn*As0vP1sm`O+n8Mi$u!J{30~s0^u%{}&ePpRaTg zH%H@!x$LQZSE-{2<_z>_LpH=SL{LHFJsF^|?ACTwyPv1$rc28e-y{DYD}nu{w;d->Mbv< zZSFQN;t7ybF0z*&f(uHw?1Az#OkSo%H22X?V!L@wqPCid`W@iwhDC;JP_24>UZ8#9 z$ZWK9#bK8FUMe>`)`BWuQPKrk?4|e-bP-7Xf!W2(+V-zVKx$#4DH&p^CeF_48jm6MO zdGOPXCB#ap4eOUs4zlT#P)4-{X2H5aX@18g#c$43G!v?`d=95 znZa!uNrcm`gKa#TWCA6#6@4v(vEuaev`R7=;>mo_bx!SbxYjs97cy*|wrPxV6Q~_u zUYGq_2=b!h-U#YAErnodWth);&hn7LC?BRovyr1M;XJ4e-NJoT89AqIyZXZ+GYBuT zrcKmY8w|DRoY~QsOZaI)Fa0+SGWn{Wv*rq4In{pT)5XNzhyT%Q`OsA5DvOIH*#6_1 zk;9==F>$xJ*bof+2dRzaIn#D>?6&>~CrY}4C$_)`r|Z9)f_TEwPpF-r3 z1>@sTR-Oh1p|xMelDF3Nf_fL{it7z4XTUWIzITAO*Q&F|hh`d2Db68`<@=)R=K?}z zy|ozEKq=ec^n8B+?XlU#{L|3;V+TLK%cr;BcKv@V zB%6T@>{LU)d`nbof;1cRxx+LYrU-jp z_j3_Mt8V6xKy?M#i=-?LfRmyB9QD$|55V%8L&9RHZry5w zGCQ7{4keCWl06}oGTD!(Cj25mxHe3D8;ASW5#|Z=Ky9=(;P`+rqPi{F`n@!cr0~mw zJBPV}4DX>Y8?Mo0H^zP$d+5aOL7FsLc{C^Tv$!Z)t8-J#1X zEP$H760MO*I#*hVs?ltYsE6J&Cp{>qF33-Fz!rt1)Ulq?&Mpg*e*7}Y>6dt1{xR_u zF-L`f)xi{>1h!m<^(RDAk^yfV3d0Fi7>MP7?`EmJN*+Umv1x;IE;fTA>t zEKOLIklD3XHiUunAbtLnoA9jq`@M-ir=c{`Gx!+7*%7i%P;`X3?i~!{Q?rjBcZp$9 zS&^NSQ6W!&)?88LNUpwIpzzm6Ha3Pi%vw;m%--Fz%Djsvk})aP0#bh?xb}l&y+f#& z=*kB~c~NO%eoQYoao#k&^CewmK+^5f;ogaA){k4NHW7%xpbGKfLw89bKmHz)zB-3< z^*~2S2}(H68hiT@8TiBgqwFyh4)f|{__*v*{dl(MC(B%z3FbPg!Exf2Dau#MA1GbM z6>zpXXLmAPiN_k@zO#<8(VyGifO0f^+nY>~MOkf6y_CoTPUoLP=N;WF^^H>jES>dV zm5!6HdvZ2+BXmmA4;06(yyEh^^?#QjP$V(p>YFXjKX#(dCL6w(6(61JAFI@WU`KiQxdn`{4NGrPDE$VjmBG;Br4aT# z(#8Yq1c8}A$ZMUD`vxI(MfcCV`#8*js;zn%AzcCfMO8Q0K!0@N%dOOvOSkkm%pj8; z7u3yolp-79dCs{5g25_0(svSY;m~o@{FVIYhRz!ud@)ex@|kI>Tmh-_*Xa7 zm=+f~E*@ZpW_)LHviNnS?jxBH#9mfbWg9@}^+e6LI7tKZ-3?Y&$l=yaUHzRmKmS(Y zv%`BLnFnD>U5ZtgeOW|fS~}!7IVhe4N!(A;vDU-Va#`I~@Ne%oL%D0Ugz59uq@mX|lisHbWq3oqt@26Z4P$X^6`(8YU2HMB357q=?H%e&7zU}RdA}B>V8l`6&vo7}{uvkpQ=YHvi;$h3kM%XQz z5~Ky(ohj9?ej7I#rK`T=#_h*Y?=#kod4x5%W$zc&gC%ZDAR%&tmW&e-UoFV`%Plo` zm!zYsD3!vAxNBbZ$1*oOshp$w^uWz5RSt!3>!9HT^0GXHbKL1Hnu#YjxG$b&M66M8;JyJz^M9#tMc z*AKU%ZQCHTVa`p}tw%~z+joMqr$;)$++DaT&e$MQO#8SPTQko zF_->-%qPgq04X+E*PpC+1mC-vkg~#83R6v?PZEDtxtNRwyWnf zr4lD>W@*KOLguVJ{}xdqQTz*833P|-Ii~G9Gg8T5x_Ma?({!0dM)J@Lp)Fvq5!=u^ z4UFYTo?r&e4ia1By4VnwJ33gK9@OsFNxvZRFJ=l{s=;s>5k;X4uJ;3>0T2_Osp`Le zAD(#BI=or#V~IC(&>Sf&P^vpGqEikYjEE``rLT_Z)@bm6i5)C_jYQpTY$6t66k_s_ zi9SBG0>siy;^ugHfW<*~pvHWNY4|VdY)#u9aRoA-w6r}U>g;TvJT4Jd;qr*|yJZxr zO-9O(BeU^!Y}Mav?KxCcxNBeW4Gw6YZ12{N5$U{hKi$;=yciIy!KsE*$`^}72?v~? zfaTgYnwpr8KZ*RgbvY}XlSE7j%A9P_9WAQJ>_`k#h6rC3+ZCxJL+LIQiG7AU`xFtb z`%GWVtLMT`a<+pdm@pxH^E0a_y5!QVuv)ix#vD3IYnFsPB{I2Y7VZ`VKX+ldZu7mB z1_7$$JgT#t_x0v(&d`^8jK9kBNY2wS9Qu8KjFwv8!|KI#5E~J!bo4t)GDI)bPzS$4 z;_xIWYU+%14SY^%Q^1(o9<@RBWlcubkH=YR<;5vy7Oij;dyD-7qz4Z%FB-&0jR=@3 z9M^OFie2-;Wz`7tGLzib4O=@_=87Nzg?QURjnkbPjn28s@-9 z=6LXMc53|_&o_e^XsHmPLCDAuL>;SW1Ywg5inasi%7)K{`_a)-A4?_}UNWfQz4N=g z03Wy!9b>ElhJ#SMrulXMOrNs@2q`78RsQ}&&8BPdrU;B-P)=VYyMulYI+7 zptmOWBh+!(I3pqDy~ABa0oJAOkmvX+V_cBrY7BA+Fr3y1LxT zW>el;0LIpW(g_KJXYS5k;ah@`u%L7aFh};No>QBqtT>J0e!>LhjaRg)NY}G&oU`3U;L{`U|yI zxfcF$`V-KQB^yS!qTYn5*^Jvt_idlW7t5Rb$QehXean;kMP_k!My=_j z=KP}R|9#r%XaAzp1m)9Tm$Ts`-{+$11=mjRHc^f`4;!xylvmo#7J<+q_|O71-Uc!( zf>Zk%!vjJrG$%K9@^XF!3Ct0SVVHZ049bJ0Uh=MWjvNv!w}1A-xL#4qb$}JI{jUj^ zwrqHT zJcJg9nB-_bMcGHY@hAnCZ#?9)pIFe!3;I-`c7P3yGbSx4+aII7TFR39OmMVUD@{|- z9zg$G(Ot0Ii04Nj3V1~%#n7@`P6ghc}Fo`Q%DtL`27X@l6 zFg$KfH2|Mq-XKkQy}tEbvTQ91}4CB?NcKN?Y?Mj}dxG-_Bk z!x-0s35gj9KhTh|U{zuKig70)74T2KH!>!NW6q|{c0;7`?W;VKIEJ#9uEFcGd{_0bc8q>0hYj;Z+UCnh;>7*-DdZHSr z+6Dm^s+Vd5^_{U$ZK@n?CZw7?T((}Ez*uPXqymiXOeqs;urWzX!#Rhnfexby5uX3F zgr@r}tFaq;Rxk-MKUVrM_G@8jg8;*g*3}(vd^(b^_=CP<2x}}3 zQ5bD1DuCRQWvpI(1Wu8Q#m!$5Y3^1xdm>tABOl7@fS2hkITc0m%BDlM?5AE zKLqU=*CK7te94MRrd;ZI5YB-9S43q1>thd?6{P>2ZyGbX0Kih%qJqzfcixI3Lt@~1Q9y$ob+oubk&cqs01Nnlt6*G6zouAmr%Uom2C@U9{kAX zW)-=yPd9_eh`MaGV+X$71-gKeG7-S{`-2I)J$z98@|`hgYUS+bX;6FP=PRl(>}ENK3tdh?R7yPI;4gGS}5X>|0X(H@HJym63%(z4I=e0 z(TZqLOZZZjHQ)5yN16Gj;|2`*i7c)>J?2t6-bRx4ZV(VFfb{zzblId9v%g6kHBvun zL^Ikr zpaS&gBkM;%QygwhChzZG%ii?Q|U+O1F6(|7(`CgHew|L|_yG$>&Vbn#5t*?tH9;gDKf>Sr!W*K5Eo`RoS{-@%A$-Za{De?8$c6^fO;Gm+&;SVLJGQumn4G7z;`7)S(wn@Y24btyi zLGmW#S1?4fB+D{A`S5A9sehmsAqX6d$Q=DTzaJCkDK^cG>+ltxzM{$X(>vQ;zwduP zW9gP}314`n0FXxHig({Na8c&o)=BspWz3BQWX=TQB%G%)6hxR|ebNHpE$Y9}&5g|# z!5yXIFCa#!bJ7ceMdxWdbV3K&1qRV7t!?bHi?W%2QO%YegOBmZtQu8$^k>H2%ug=$ zfE&?c?kqSo#&*mx`4M$3oFv8`XD}1`s9!rxcpWS?OWkzUrs+xtYyQ6J?1kUCeW*>y z$)`hxJ#4XOU*AXfywjchjZeO3tLy)TtIf4yKE|L!Nvx*}v}O~E7D-IX<2f^C5VN`L zIVdleQai5wgHaeBRVO2(+_Nms!ehG~wcv0tM}SpZ?_W{kWaA~ytq`160s2HZ8FGoO zkILgDB9srILeLx?yo7i99CThotyow|r0iNP4znXQGN83O4{72=?crbV2*gYvuU@jx zByEx%6oW4TrF=pBbxbCNm^+Q{EgdJn@OSO{!K+xw265&XnHfrky*5>UsBRvX&Zq0r|5|@HYv)0rDSu>-yDAU&gTQ)0b1kHTvv%yuz}% zJim(7lmoZ6Qutn)bBk@An3aCC6ALP&M^#EPRfW&H9Lk%yr!N5$E(=GbszsxQEbuY+ z3~VM(eCTHqzTrd0s%nh}Yr+ma?Uru<-u9e&jWGGHR==Zr#8Ktk`?WnQ567$EwW|K* z=272vimulaAmbtY_dP0s8pmGp*GkfW?UN+?U)#^)#NyW9ydz3+Cn*>|mH!)fIpZ>B z!>H~&Qs&h46i$GBTXRVcrL>y+3fAB1b6bzj+wLo0_uWyZ6nF_KJ)(VOa9d6P2IT!P zet|iaPpcu(H|b@^Kb_9ugDQSzNQ6?c!Mz!+9acqbGI`i$iC@Ft(!$O8{^7v;EY>Mf zRy1;B;Am-!TGmre?Sh8N^XazpG^`V|;#y;G0@dV)wHClEaNaF=bAE%E>t~+GHkyo_ zw@-Aeh**eYF`jMpi7;y%ZF7&8PCXGXb_PlQ$$T|%8myBw2mb(ubbKrBQZo;cw?e9;K=ZpoFxV;Fr|}$3PM( zEIXt-)}YdY*Fht?C~neNFcXh+`GV>^Rs(}{6NwZbEAaF5U9Fh`j~0-)>vZ>}f||30 z8g`QUQ8mO)3rN^x&tK3DDFBS|$>)1j=CLHGk+#^X#fQP@jvsmUDom1x@b;8S>TUVz zmzE4AqvT6xszesUzo9Zi^RScF0|%PL%=R6tk-4^~19u5&uOc~bpIMeQ- pC;#Wk?*E_Vq5qXmITMi{{c>c)6VN9um4MevQC3Z+O3FOozW|ym?BW0b literal 0 HcmV?d00001 diff --git a/redash/query_runner/dynamodb.py b/redash/query_runner/dynamodb.py new file mode 100644 index 0000000000..a66802e9a2 --- /dev/null +++ b/redash/query_runner/dynamodb.py @@ -0,0 +1,182 @@ +import logging +import sys +import boto3 + +from redash.query_runner import * +from redash.utils import json_dumps, json_loads + +import pandas as pd + +logger = logging.getLogger(__name__) + +try: + from dql import Engine, FragmentEngine + from dynamo3 import DynamoDBError + from pyparsing import ParseException + enabled = True +except ImportError as e: + enabled = False + +TYPES_MAP = { + "bool": TYPE_BOOLEAN, + "datetime64[ns]": TYPE_DATETIME, + "datetime64[s]": TYPE_DATETIME, + "float64": TYPE_FLOAT, + "int64": TYPE_INTEGER, + "object": TYPE_STRING +} + + +class DynamoDB(BaseSQLQueryRunner): + should_annotate_query = False + + @classmethod + def configuration_schema(cls): + return { + "type": "object", + "properties": { + "region": { + "type": "string", + "default": "us-east-1" + }, + "aws_iam_role_arn": {"type": "string", "title": "IAM Role ARN"}, + }, + "required": ["aws_iam_role_arn"], + } + + def _get_client(self): + sts = boto3.client('sts') + response = sts.assume_role( + RoleArn=self.configuration.get('aws_iam_role_arn'), + RoleSessionName="redash-session" + ) + dynamodb = boto3.client( + "dynamodb", + region_name=self.configuration.get("region"), + aws_access_key_id=response['Credentials']['AccessKeyId'], + aws_secret_access_key=response['Credentials']['SecretAccessKey'], + aws_session_token=response["Credentials"]["SessionToken"] + ) + logger.info("-----------DynamoDB client created------------") + return dynamodb + + def test_connection(self): + dynamodb_client = self._get_client() + dynamodb_client.close() + + + @classmethod + def type(cls): + return "dynamodb" + + @classmethod + def name(cls): + return "DynamoDB" + + # def _connect(self): + # sts = boto3.client('sts') + # response = sts.assume_role( + # RoleArn=self.configuration.get('aws_iam_role_arn'), + # RoleSessionName="redash-session" + # ) + + # engine = FragmentEngine() + # logger.info("-----------FragmentEngine Created------------") + + # config = self.configuration.to_dict() + + # if not config.get('region'): + # config['region'] = 'us-east-1' + + # if config.get('host') == '': + # config['host'] = None + + # config['access_key'] = response['Credentials']['AccessKeyId'] + # config['secret_key'] = response['Credentials']['SecretAccessKey'] + # config.pop('aws_iam_role_arn') + + # engine.connect(**config) + # logger.info("-----------FragmentEngine Connected to DynamoDB------------") + + # return engine + + # def _get_tables(self, schema): + # engine = self._connect() + + # # We can't use describe_all because sometimes a user might give List permission + # # for * (all tables), but describe permission only for some of them. + # tables = engine.connection.list_tables() + # for table_name in tables: + # try: + # table = engine.describe(table_name, True) + # schema[table.name] = {'name': table.name, + # 'columns': table.attrs.keys()} + # except DynamoDBError: + # pass + + def run_query(self, query, user): + dynamodb_client = None + try: + dynamodb_client = self._get_client() + + # if not query.endswith(';'): + # query = query + ';' + + result = dynamodb_client.execute_statement(Statement=query) + logger.info("----------------Query has been executed!-----------------") + logger.info("JSON Dump: %s", json_dumps(result)) + df = pd.DataFrame([i.decode('utf-8') for i in results['Items']]) + logger.info("DataFrame: %s", df.to_string()) + + columns = [] + rows = df.to_dict('records') + + for col in df.columns: + columns.append( + { + "name": col, + "friendly_name": col, + "type": TYPES_MAP[str(df[col].dtype)] + } + ) + + # # When running a count query it returns the value as a string, in which case + # # we transform it into a dictionary to be the same as regular queries. + # if isinstance(result, basestring): + # # when count < scanned_count, dql returns a string with number of rows scanned + # value = result.split(" (")[0] + # if value: + # value = int(value) + # result = [{"value": value}] + + # for item in result: + # if not columns: + # for k, v in item.iteritems(): + # columns.append({ + # 'name': k, + # 'friendly_name': k, + # 'type': types_map.get(str(type(v)).upper(), None) + # }) + # rows.append(item) + + # Returning the query results in Redash format + data = {"columns": columns, "rows": rows} + error = None + json_data = json_dumps(data) + except ParseException as e: + error = u"Error parsing query at line {} (column {}):\n{}".format(e.lineno, e.column, e.line) + json_data = None + except (SyntaxError, RuntimeError) as e: + error = e.message + json_data = None + except KeyboardInterrupt: + if engine and engine.connection: + engine.connection.cancel() + error = "Query cancelled by user." + json_data = None + finally: + dynamodb_client.close() + return json_data, error + + +register(DynamoDB) \ No newline at end of file diff --git a/redash/settings/__init__.py b/redash/settings/__init__.py index 7d44a3e80b..cbf162dace 100644 --- a/redash/settings/__init__.py +++ b/redash/settings/__init__.py @@ -299,7 +299,7 @@ def email_server_is_configured(): "redash.query_runner.vertica", "redash.query_runner.clickhouse", "redash.query_runner.tinybird", - "redash.query_runner.yandex_metrica", + #"redash.query_runner.yandex_metrica", "redash.query_runner.yandex_disk", "redash.query_runner.rockset", "redash.query_runner.treasuredata", @@ -340,6 +340,7 @@ def email_server_is_configured(): "redash.query_runner.ignite", "redash.query_runner.oracle", "redash.query_runner.e6data", + "redash.query_runner.dynamodb" ] enabled_query_runners = array_from_string(