From f56965d4cd6b66f9ec682f1015672114bdd3eb75 Mon Sep 17 00:00:00 2001 From: "Bradley W. Settlemyer" Date: Wed, 29 Jan 2014 15:13:46 -0500 Subject: [PATCH 01/50] Add required Python packages Adding local versions of the Python packages leveraged by XDD's xddmcp copy tool. --- contrib/site-packages/Pyro4/__init__.py | 65 + contrib/site-packages/Pyro4/__init__.pyo | Bin 0 -> 2660 bytes contrib/site-packages/Pyro4/configuration.py | 128 + contrib/site-packages/Pyro4/configuration.pyo | Bin 0 -> 5684 bytes contrib/site-packages/Pyro4/constants.py | 20 + contrib/site-packages/Pyro4/constants.pyo | Bin 0 -> 533 bytes contrib/site-packages/Pyro4/core.py | 934 +++++++ contrib/site-packages/Pyro4/core.pyo | Bin 0 -> 45729 bytes contrib/site-packages/Pyro4/errors.py | 48 + contrib/site-packages/Pyro4/errors.pyo | Bin 0 -> 2923 bytes contrib/site-packages/Pyro4/futures.py | 147 ++ contrib/site-packages/Pyro4/futures.pyo | Bin 0 -> 7569 bytes contrib/site-packages/Pyro4/message.py | 193 ++ contrib/site-packages/Pyro4/message.pyo | Bin 0 -> 8857 bytes contrib/site-packages/Pyro4/naming.py | 384 +++ contrib/site-packages/Pyro4/naming.pyo | Bin 0 -> 16892 bytes contrib/site-packages/Pyro4/nsc.py | 102 + contrib/site-packages/Pyro4/nsc.pyo | Bin 0 -> 5439 bytes .../Pyro4/socketserver/__init__.py | 5 + .../Pyro4/socketserver/__init__.pyo | Bin 0 -> 349 bytes .../Pyro4/socketserver/multiplexserver.py | 223 ++ .../Pyro4/socketserver/multiplexserver.pyo | Bin 0 -> 9764 bytes .../Pyro4/socketserver/threadpoolserver.py | 188 ++ .../Pyro4/socketserver/threadpoolserver.pyo | Bin 0 -> 9070 bytes contrib/site-packages/Pyro4/socketutil.py | 520 ++++ contrib/site-packages/Pyro4/socketutil.pyo | Bin 0 -> 18933 bytes contrib/site-packages/Pyro4/test/__init__.py | 1 + contrib/site-packages/Pyro4/test/__init__.pyo | Bin 0 -> 197 bytes .../site-packages/Pyro4/test/echoserver.py | 126 + .../site-packages/Pyro4/test/echoserver.pyo | Bin 0 -> 6168 bytes contrib/site-packages/Pyro4/threadutil.py | 19 + contrib/site-packages/Pyro4/threadutil.pyo | Bin 0 -> 801 bytes contrib/site-packages/Pyro4/tpjobqueue.py | 203 ++ contrib/site-packages/Pyro4/tpjobqueue.pyo | Bin 0 -> 10078 bytes contrib/site-packages/Pyro4/util.py | 546 +++++ contrib/site-packages/Pyro4/util.pyo | Bin 0 -> 26310 bytes contrib/site-packages/Pyro4/utils/__init__.py | 1 + .../site-packages/Pyro4/utils/__init__.pyo | Bin 0 -> 198 bytes contrib/site-packages/Pyro4/utils/flame.py | 299 +++ contrib/site-packages/Pyro4/utils/flame.pyo | Bin 0 -> 17806 bytes .../site-packages/Pyro4/utils/flameserver.py | 55 + .../site-packages/Pyro4/utils/flameserver.pyo | Bin 0 -> 2471 bytes contrib/site-packages/paramiko/__init__.py | 143 ++ contrib/site-packages/paramiko/_winapi.py | 269 ++ contrib/site-packages/paramiko/agent.py | 380 +++ .../site-packages/paramiko/auth_handler.py | 426 ++++ contrib/site-packages/paramiko/ber.py | 129 + .../site-packages/paramiko/buffered_pipe.py | 200 ++ contrib/site-packages/paramiko/channel.py | 1279 ++++++++++ contrib/site-packages/paramiko/client.py | 538 ++++ contrib/site-packages/paramiko/common.py | 129 + contrib/site-packages/paramiko/compress.py | 39 + contrib/site-packages/paramiko/config.py | 266 ++ contrib/site-packages/paramiko/dsskey.py | 196 ++ contrib/site-packages/paramiko/ecdsakey.py | 181 ++ contrib/site-packages/paramiko/file.py | 460 ++++ contrib/site-packages/paramiko/hostkeys.py | 342 +++ contrib/site-packages/paramiko/kex_gex.py | 243 ++ contrib/site-packages/paramiko/kex_group1.py | 135 ++ contrib/site-packages/paramiko/logging22.py | 66 + contrib/site-packages/paramiko/message.py | 302 +++ contrib/site-packages/paramiko/packet.py | 500 ++++ contrib/site-packages/paramiko/pipe.py | 147 ++ contrib/site-packages/paramiko/pkey.py | 381 +++ contrib/site-packages/paramiko/primes.py | 151 ++ contrib/site-packages/paramiko/proxy.py | 91 + contrib/site-packages/paramiko/resource.py | 72 + contrib/site-packages/paramiko/rsakey.py | 185 ++ contrib/site-packages/paramiko/server.py | 669 +++++ contrib/site-packages/paramiko/sftp.py | 188 ++ contrib/site-packages/paramiko/sftp_attr.py | 223 ++ contrib/site-packages/paramiko/sftp_client.py | 787 ++++++ contrib/site-packages/paramiko/sftp_file.py | 489 ++++ contrib/site-packages/paramiko/sftp_handle.py | 202 ++ contrib/site-packages/paramiko/sftp_server.py | 444 ++++ contrib/site-packages/paramiko/sftp_si.py | 310 +++ .../site-packages/paramiko/ssh_exception.py | 132 + contrib/site-packages/paramiko/transport.py | 2158 +++++++++++++++++ contrib/site-packages/paramiko/util.py | 311 +++ contrib/site-packages/paramiko/win_pageant.py | 125 + 80 files changed, 17495 insertions(+) create mode 100644 contrib/site-packages/Pyro4/__init__.py create mode 100644 contrib/site-packages/Pyro4/__init__.pyo create mode 100644 contrib/site-packages/Pyro4/configuration.py create mode 100644 contrib/site-packages/Pyro4/configuration.pyo create mode 100644 contrib/site-packages/Pyro4/constants.py create mode 100644 contrib/site-packages/Pyro4/constants.pyo create mode 100644 contrib/site-packages/Pyro4/core.py create mode 100644 contrib/site-packages/Pyro4/core.pyo create mode 100644 contrib/site-packages/Pyro4/errors.py create mode 100644 contrib/site-packages/Pyro4/errors.pyo create mode 100644 contrib/site-packages/Pyro4/futures.py create mode 100644 contrib/site-packages/Pyro4/futures.pyo create mode 100644 contrib/site-packages/Pyro4/message.py create mode 100644 contrib/site-packages/Pyro4/message.pyo create mode 100644 contrib/site-packages/Pyro4/naming.py create mode 100644 contrib/site-packages/Pyro4/naming.pyo create mode 100644 contrib/site-packages/Pyro4/nsc.py create mode 100644 contrib/site-packages/Pyro4/nsc.pyo create mode 100644 contrib/site-packages/Pyro4/socketserver/__init__.py create mode 100644 contrib/site-packages/Pyro4/socketserver/__init__.pyo create mode 100644 contrib/site-packages/Pyro4/socketserver/multiplexserver.py create mode 100644 contrib/site-packages/Pyro4/socketserver/multiplexserver.pyo create mode 100644 contrib/site-packages/Pyro4/socketserver/threadpoolserver.py create mode 100644 contrib/site-packages/Pyro4/socketserver/threadpoolserver.pyo create mode 100644 contrib/site-packages/Pyro4/socketutil.py create mode 100644 contrib/site-packages/Pyro4/socketutil.pyo create mode 100644 contrib/site-packages/Pyro4/test/__init__.py create mode 100644 contrib/site-packages/Pyro4/test/__init__.pyo create mode 100644 contrib/site-packages/Pyro4/test/echoserver.py create mode 100644 contrib/site-packages/Pyro4/test/echoserver.pyo create mode 100644 contrib/site-packages/Pyro4/threadutil.py create mode 100644 contrib/site-packages/Pyro4/threadutil.pyo create mode 100644 contrib/site-packages/Pyro4/tpjobqueue.py create mode 100644 contrib/site-packages/Pyro4/tpjobqueue.pyo create mode 100644 contrib/site-packages/Pyro4/util.py create mode 100644 contrib/site-packages/Pyro4/util.pyo create mode 100644 contrib/site-packages/Pyro4/utils/__init__.py create mode 100644 contrib/site-packages/Pyro4/utils/__init__.pyo create mode 100644 contrib/site-packages/Pyro4/utils/flame.py create mode 100644 contrib/site-packages/Pyro4/utils/flame.pyo create mode 100644 contrib/site-packages/Pyro4/utils/flameserver.py create mode 100644 contrib/site-packages/Pyro4/utils/flameserver.pyo create mode 100644 contrib/site-packages/paramiko/__init__.py create mode 100644 contrib/site-packages/paramiko/_winapi.py create mode 100644 contrib/site-packages/paramiko/agent.py create mode 100644 contrib/site-packages/paramiko/auth_handler.py create mode 100644 contrib/site-packages/paramiko/ber.py create mode 100644 contrib/site-packages/paramiko/buffered_pipe.py create mode 100644 contrib/site-packages/paramiko/channel.py create mode 100644 contrib/site-packages/paramiko/client.py create mode 100644 contrib/site-packages/paramiko/common.py create mode 100644 contrib/site-packages/paramiko/compress.py create mode 100644 contrib/site-packages/paramiko/config.py create mode 100644 contrib/site-packages/paramiko/dsskey.py create mode 100644 contrib/site-packages/paramiko/ecdsakey.py create mode 100644 contrib/site-packages/paramiko/file.py create mode 100644 contrib/site-packages/paramiko/hostkeys.py create mode 100644 contrib/site-packages/paramiko/kex_gex.py create mode 100644 contrib/site-packages/paramiko/kex_group1.py create mode 100644 contrib/site-packages/paramiko/logging22.py create mode 100644 contrib/site-packages/paramiko/message.py create mode 100644 contrib/site-packages/paramiko/packet.py create mode 100644 contrib/site-packages/paramiko/pipe.py create mode 100644 contrib/site-packages/paramiko/pkey.py create mode 100644 contrib/site-packages/paramiko/primes.py create mode 100644 contrib/site-packages/paramiko/proxy.py create mode 100644 contrib/site-packages/paramiko/resource.py create mode 100644 contrib/site-packages/paramiko/rsakey.py create mode 100644 contrib/site-packages/paramiko/server.py create mode 100644 contrib/site-packages/paramiko/sftp.py create mode 100644 contrib/site-packages/paramiko/sftp_attr.py create mode 100644 contrib/site-packages/paramiko/sftp_client.py create mode 100644 contrib/site-packages/paramiko/sftp_file.py create mode 100644 contrib/site-packages/paramiko/sftp_handle.py create mode 100644 contrib/site-packages/paramiko/sftp_server.py create mode 100644 contrib/site-packages/paramiko/sftp_si.py create mode 100644 contrib/site-packages/paramiko/ssh_exception.py create mode 100644 contrib/site-packages/paramiko/transport.py create mode 100644 contrib/site-packages/paramiko/util.py create mode 100644 contrib/site-packages/paramiko/win_pageant.py diff --git a/contrib/site-packages/Pyro4/__init__.py b/contrib/site-packages/Pyro4/__init__.py new file mode 100644 index 00000000..4dca7320 --- /dev/null +++ b/contrib/site-packages/Pyro4/__init__.py @@ -0,0 +1,65 @@ +""" +Pyro package. Some generic init stuff to set up logging etc. + +Pyro - Python Remote Objects. Copyright by Irmen de Jong (irmen@razorvine.net). +""" + +import sys + +if sys.version_info < (2, 6): + import warnings + warnings.warn("This Pyro version is unsupported on Python versions older than 2.6", ImportWarning) + + +def _configLogging(): + """Do some basic config of the logging module at package import time. + The configuring is done only if the PYRO_LOGLEVEL env var is set. + If you want to use your own logging config, make sure you do + that before any Pyro imports. Then Pyro will skip the autoconfig. + Set the env var PYRO_LOGFILE to change the name of the autoconfigured + log file (default is pyro.log in the current dir). Use '{stderr}' to + make the log go to the standard error output.""" + import os, logging + level = os.environ.get("PYRO_LOGLEVEL") + logfilename = os.environ.get("PYRO_LOGFILE", "pyro.log") + if logfilename == "{stderr}": + logfilename = None + if level is not None: + levelvalue = getattr(logging, level) + if len(logging.root.handlers) == 0: + # configure the logging with some sensible defaults. + try: + import tempfile + tempfile = tempfile.TemporaryFile(dir=".") + tempfile.close() + except OSError: + # cannot write in current directory, use the default console logger + logging.basicConfig(level=levelvalue) + else: + # set up a basic logfile in current directory + logging.basicConfig( + level=levelvalue, + filename=logfilename, + datefmt="%Y-%m-%d %H:%M:%S", + format="[%(asctime)s.%(msecs)03d,%(name)s,%(levelname)s] %(message)s" + ) + log = logging.getLogger("Pyro4") + log.info("Pyro log configured using built-in defaults, level=%s", level) + else: + # PYRO_LOGLEVEL is not set, disable Pyro logging. No message is printed about this fact. + log = logging.getLogger("Pyro4") + log.setLevel(9999) + +_configLogging() +del _configLogging + +# initialize Pyro's configuration +from Pyro4.configuration import Configuration +config=Configuration() +del Configuration + +# import the required Pyro symbols into this package +from Pyro4.core import URI, Proxy, Daemon, callback, batch, async +from Pyro4.naming import locateNS, resolve +from Pyro4.futures import Future +from Pyro4.constants import VERSION as __version__ diff --git a/contrib/site-packages/Pyro4/__init__.pyo b/contrib/site-packages/Pyro4/__init__.pyo new file mode 100644 index 0000000000000000000000000000000000000000..b38df93fbd30e0631ad3bb0d04060440559061ad GIT binary patch literal 2660 zcmc&#{camY5TErWj_rInP1;hVQnnNmM@r79d`VHMD%u2sLQ-AQmbwD#>{~nM+})bp zy#zz@S9t(lfp;3P9<)7B9t>w7ngulruS*~@Tv>)+Fhm(t8D0a3 zS7lx?2gw|y6=+vLs0vxyRd`ilTLqFDr1Q|8hjanj3y>~CdlAwlXfHuphjty(WoR#h zAXrsMu0gs2?G;E@p}ksYYmi)r^aiwV6!JVIHz9os+P6?H!C`Ze68sg1+S9SoauoJX z!h!PTp-zp zbcd?vtS{wbJsO*MaO`AvEO$++G8rlPwZ`?$n8ZgW{7IYBI8%P6+`3$zTDe>zP%?^b}COpL!L;_ zBFb)Gj&&~2!VF_;nOjA&k@_sV1XJv?A=B_gNt+vX!Oa*3at8UT`Wn??HZF))fJIIt zNFp0@@hnafX;0!2=NaZs7hww+hj?5l|0%>N@SWYgZNk+<{ zc;aL~#@aO_)erN;QM!1pv`>~e;~2f%7=;-}v03-!GfaBzXY23?n2R;sjstPFlb*y*1u?)>5)I=U#P(7EPFzAFtSV=Jhb!eL>$af}9V9uA1gAy2@^ z69JH@Q`2# z(DsNs>(*~HQ>%J*{lkxUUKV}K~M`nwx3`qZYGs-%00YL za!Um4=5gXSvBHzW+YQMvKX2H$z6Q`FsE+O`Z8?R><;Zhvd~9^a`45zHGw8ODbY>$e zLgx${u>@>1T05=^G9yaxqGQ3SbVOz1P8^}2O=!kDq@h~&5{=BK@*Ny*Q~%sN&iNod z=5y%i#>4IQ@ZMI2H;S{q=GeG$k9yIh^#M5sH5v#i$H9CqTSOMl{xnQ-6|9gC^%i}a z_y;^#>Um3#Tr1JNFlp_@U4q-%G1!zdZBA^fxTyX9RMUE1-08M;-~X(Y#NF12UlWpS z?9}GuaiW#jvR0=`<(#0o~ zJ-VUm9rG--F zqLIDRiFB{iVQ_YA`B+XKKP=Fs!Dpe#uqc+>P6DU6%NOs9!WO(k5$P)D^7_+5!1)Jv z$Vl)CwIdQt{fr9UMKO>x+@;|?8t&2XJ`TlAK|S;F5F(7AL8cGL#McU&`kc27)<|8) zVH!T8d&${=c*SJwbf)=rt_b&ku&#oL9u)7{Lpp=@HV);|0&;0Vta{b8IpKNB$UD#T J#Z9p+{smq5Sc8e$7XM5n}7DxuS2D=pl#@FqkX+zIiW5Fdot z+?o_#M!1t=FD1e^!keVl&J>l;p7$|rN;L8rT51nPt{C*&{?0ITv>)_Mjl~~Yug>KrPYdQ7K6bk^mlf(xjiz=VbANEu4jG|^moj>Pu0hv z^V=Xi@cZ6M-_y5Oa{j;ZSJg9E^`k{wXyCugD4*b~9-{D|1BmqnOMIDxgSe2@y&}GY zN+yMSl^Y@IH72HnYcg?4$VUD;rqB~8p4S>pJq4oLu*d0oO*VBlYJXDvxW@fXgIBDr z*QM2HlxtO;L%m_iSC(vU)-6r!QmxsvO4Y);)!;h1uQ$u-ActlGJH<^SQJN zrE>MzJw1zRt!ix)Htnd@D(TN?N~>8YSFDoVl!c=8q)>bjEnF+Uu%-2~Vd=Bzg-t!R zq+PDt*beXR{L%FFTCHNQm#fjV4SkYRc%5+OKOR#qRjg#Ur@(r?UMSixtWDNZ;bpT{ zm$lcMyz?irRwxwLcuk}YB12N-Js=kS1bN@=!B!yr=ipH-Ui{GT8&&` zrJb~_zVbo^uTiV_R=!^C`rE4mb{48BUwd~3PHWHE@n~cie6`vdceNGtB`w>w-}kj` zUjnUCD1_0XA2M0;QkLxrS^xnM&rXCSnn8gGfI^zNK!gP*;2#kI|Cr#GDS`EOj`x|jP*F@>fU&SFLPl@+waY37+ z^M%;Ui156juTvMaen^SK2@$d4ehY_?A4sww_Oc@U-q1Iw`oR!~&_YJ&n_};j2>)!n zpA?;2gqzeK3cbX0|IUCO(X#&+>fkPN;~x!ZCu-C`5$=4vC3I+~#XP3|))4#O8<-tU zw9=%2z3HWRlhl|N#{PNw-vEcin-}4Y6zz>^4AObt@QqZ|(zz|%3p5$3_B_w`Q|^34 zbbcn>i^Ph#S4h}K{xa-I{T@ZmjQ<**25h&TVON`xd73cX9`HZuQEr<<<$3)BKMeYS zb#3~}eCl+Sw`{%*`b*jjJ;yc2I~+Kn?`(HHbrep_JmzQ*yQbeZN5Rl+IW$m*L#3%6 z(r`qGWKBjKcEjC3Y5&_BLiiu3z}AH(uh6@+>JHWEFCiZLNLur-PYuq{eanRQUbHT zWcJ8Y%*JTi1*G%TRS!(@?kX{DP-iuI6d5 zyoo}^i!vQfN`kth9C4~Ta0R>CM z$0|v`lHf-YK1qs^lA@O+gq0NcSiUCYlhf2h5nU40ONvdB!k9cq1s#FBNCgEUxj+R4 zK1pCF7pWkV}^;%&V zgz-s7!${oik;WO3d|tm5xE>!Hr98w`gRZahEH|B+CTTymd>%=mxBQw(HR^50%k!Bs zo@2{Vh4dCt*er(_E#h<7&`ppfRgSs);X$pgFq%Rp%WJczaZIk5pU z3f_u5IrQ`}>?>36dS+`FhJ-;eRyoRafImCKt`nNB-=Z8AK|9Yje-~rr&r#&r`uR}l z?2!fl?$~??>ptnxupK$7gss2E))aZ1U^QQe(g0E-ZJ!k>K>+F~0$QXmao!Lk2A zr0@a2-;Gd|&4PUONKi?-X7iw?Pir28pirao3du1v_!N)^Mj5Y|%oy|Rjahgl`ySa4 z!5Ev&J$Pe$c@(0D!~+LUbs@S)jNpN35)}pl4-D@U-Qf!(*~=ouTHG9wK+;{ZMtNf& zW|d+4LRNB$Z4KEKx@AQtN9i9di*l<|V(qv!o3!S{8l`xKm`8#~{L`XyM9i>xBH^5+ z?)XQtw}78{_Y7GxZEcPgB;qU+yu0=Udd!NQ2}-KoeJyZ}!Uj6$s0T)u-rNwsf~eRu zQd?wnNK%o=A`zPs?G)!`#{R#=hI>|Y&d1qVBTq4`$`{t?zCijMziLH^zU}u0U5}C? zzCB}OP>OTlKr$h|;ky6_=tqwpX^NDbP~Zad6gNB$P6<4B93^b` z6-kUC571u`qUZ9IEAZ`)Z!;?xsI-F|N>+$L{ecGqJP$Cv>nIM_WR37%-44Q@Tt`Jx zIAB%vakG*ZH4)5nFh-DP*1*xbJdLxhh?eZdv<7;#{jNt)7{%!LC}L4r^=hO5>=B7H zz;oIEIkA<`X&fI^;P>;iJ4p&-A>_+s(h~L; ze5xWf2Q-*9W>bs!oyTX&ST?RBN?$V;jpznIgXpTxXB*!B9Ts`XOV`hr$|__ z3qeX!bWzA4WQ~6^scd@sm)WW8Oe|;*iHL2pFEeJ?wi~n{Dg#ZLrWq-@fnP?1fS)iK fKjaP?|4(FQS=ORsddT+?iWr#;C)(-xVA}W(-X;>^ literal 0 HcmV?d00001 diff --git a/contrib/site-packages/Pyro4/constants.py b/contrib/site-packages/Pyro4/constants.py new file mode 100644 index 00000000..546d39e7 --- /dev/null +++ b/contrib/site-packages/Pyro4/constants.py @@ -0,0 +1,20 @@ +""" +Definitions of various hard coded constants. + +Pyro - Python Remote Objects. Copyright by Irmen de Jong (irmen@razorvine.net). +""" + +# Pyro version +VERSION = "4.22" + +# standard object name for the Daemon object +DAEMON_NAME = "Pyro.Daemon" + +# standard name for the Name server itself +NAMESERVER_NAME = "Pyro.NameServer" + +# standard name for Flame server +FLAME_NAME = "Pyro.Flame" + +# wire protocol version +PROTOCOL_VERSION = 46 diff --git a/contrib/site-packages/Pyro4/constants.pyo b/contrib/site-packages/Pyro4/constants.pyo new file mode 100644 index 0000000000000000000000000000000000000000..544089b24c05e4ab04b466f1dbdfbb0c55579eb7 GIT binary patch literal 533 zcmYLFOK!q26g1yfsjBt}UZx9T2_bb;RUkeR6(lHBEV5{1VgrU?Bim4=t6rsN>k)c^ zKJyWdJb5#Ip1e%u@80|P{fRSlUL|<9VH+QSA(Q}?P*OsyjFK{96_ivEtD>ZeSPdmL z#Oi43-?$s!0A7zTgcMdNX|P)01Jy$94PH^raHbe{D2=7kn!xiWg;uzMCxu-piDSN1 zmg8u?<{2Dg+*P|mi{;AVd4Y#|%Oz$UzbU!IzHqdy>6g+6A$cIVeF{8Ng&|r&vuW<2 zaP~ouf~_= (3, 0): + basestring = str + +log = logging.getLogger("Pyro4.core") + + +class URI(object): + """ + Pyro object URI (universal resource identifier). + The uri format is like this: ``PYRO:objectid@location`` where location is one of: + + - ``hostname:port`` (tcp/ip socket on given port) + - ``./u:sockname`` (Unix domain socket on localhost) + + There is also a 'Magic format' for simple name resolution using Name server: + ``PYRONAME:objectname[@location]`` (optional name server location, can also omit location port) + + You can write the protocol in lowercase if you like (``pyro:...``) but it will + automatically be converted to uppercase internally. + """ + uriRegEx = re.compile(r"(?P[Pp][Yy][Rr][Oo][a-zA-Z]*):(?P\S+?)(@(?P\S+))?$") + __slots__ = ("protocol", "object", "sockname", "host", "port", "object") + + def __init__(self, uri): + if isinstance(uri, URI): + state=uri.__getstate__() + self.__setstate__(state) + return + if not isinstance(uri, basestring): + raise TypeError("uri parameter object is of wrong type") + self.sockname=self.host=self.port=None + match=self.uriRegEx.match(uri) + if not match: + raise errors.PyroError("invalid uri") + self.protocol=match.group("protocol").upper() + self.object=match.group("object") + location=match.group("location") + if self.protocol=="PYRONAME": + self._parseLocation(location, Pyro4.config.NS_PORT) + return + if self.protocol=="PYRO": + if not location: + raise errors.PyroError("invalid uri") + self._parseLocation(location, None) + else: + raise errors.PyroError("invalid uri (protocol)") + + def _parseLocation(self, location, defaultPort): + if not location: + return + if location.startswith("./u:"): + self.sockname=location[4:] + if (not self.sockname) or ':' in self.sockname: + raise errors.PyroError("invalid uri (location)") + else: + if location.startswith("["): # ipv6 + if location.startswith("[["): # possible mistake: double-bracketing + raise errors.PyroError("invalid ipv6 address: enclosed in too many brackets") + self.host, _, self.port = re.match(r"\[([0-9a-fA-F:%]+)](:(\d+))?", location).groups() + else: + self.host, _, self.port = location.partition(":") + if not self.port: + self.port=defaultPort + try: + self.port=int(self.port) + except (ValueError, TypeError): + raise errors.PyroError("invalid port in uri, port="+str(self.port)) + + @staticmethod + def isUnixsockLocation(location): + """determine if a location string is for a Unix domain socket""" + return location.startswith("./u:") + + @property + def location(self): + """property containing the location string, for instance ``"servername.you.com:5555"``""" + if self.host: + if ":" in self.host: # ipv6 + return "[%s]:%d" % (self.host, self.port) + else: + return "%s:%d" % (self.host, self.port) + elif self.sockname: + return "./u:"+self.sockname + else: + return None + + def asString(self): + """the string representation of this object""" + result=self.protocol+":"+self.object + location=self.location + if location: + result+="@"+location + return result + + def __str__(self): + string=self.asString() + if sys.version_info<(3, 0) and type(string) is unicode: + return string.encode("ascii", "replace") + return string + + def __unicode__(self): + return self.asString() + + def __repr__(self): + return "<%s.%s at 0x%x, %s>" % (self.__class__.__module__, self.__class__.__name__, id(self), str(self)) + + def __eq__(self, other): + if not isinstance(other, URI): + return False + return (self.protocol, self.object, self.sockname, self.host, self.port) \ + == (other.protocol, other.object, other.sockname, other.host, other.port) + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((self.protocol, self.object, self.sockname, self.host, self.port)) + + # note: getstate/setstate are not needed if we use pickle protocol 2, + # but this way it helps pickle to make the representation smaller by omitting all attribute names. + + def __getstate__(self): + return self.protocol, self.object, self.sockname, self.host, self.port + + def __getstate_for_dict__(self): + return self.__getstate__() + + def __setstate__(self, state): + self.protocol, self.object, self.sockname, self.host, self.port = state + + +class _RemoteMethod(object): + """method call abstraction""" + def __init__(self, send, name): + self.__send = send + self.__name = name + + def __getattr__(self, name): + return _RemoteMethod(self.__send, "%s.%s" % (self.__name, name)) + + def __call__(self, *args, **kwargs): + return self.__send(self.__name, args, kwargs) + + +def _check_hmac(): + if Pyro4.config.HMAC_KEY: + if sys.version_info>=(3, 0) and type(Pyro4.config.HMAC_KEY) is not bytes: + raise errors.PyroError("HMAC_KEY must be bytes type") + + +class Proxy(object): + """ + Pyro proxy for a remote object. Intercepts method calls and dispatches them to the remote object. + + .. automethod:: _pyroBind + .. automethod:: _pyroRelease + .. automethod:: _pyroReconnect + .. automethod:: _pyroBatch + .. automethod:: _pyroAsync + """ + __pyroAttributes=frozenset(["__getnewargs__", "__getinitargs__", "_pyroConnection", "_pyroUri", "_pyroOneway", "_pyroTimeout", "_pyroSeq"]) + + def __init__(self, uri): + """ + .. autoattribute:: _pyroOneway + .. autoattribute:: _pyroTimeout + """ + _check_hmac() # check if hmac secret key is set + if isinstance(uri, basestring): + uri=URI(uri) + elif not isinstance(uri, URI): + raise TypeError("expected Pyro URI") + self._pyroUri=uri + self._pyroConnection=None + self._pyroOneway=set() + self._pyroSeq=0 # message sequence number + self.__pyroTimeout=Pyro4.config.COMMTIMEOUT + self.__pyroLock=threadutil.Lock() + self.__pyroConnLock=threadutil.Lock() + util.get_serializer(Pyro4.config.SERIALIZER) # assert that the configured serializer is available + if os.name=="java" and Pyro4.config.SERIALIZER=="marshal": + import warnings + warnings.warn("marshal doesn't work correctly with Jython (issue 2077); please choose another serializer", RuntimeWarning) + + def __del__(self): + if hasattr(self, "_pyroConnection"): + self._pyroRelease() + + def __getattr__(self, name): + if name in Proxy.__pyroAttributes: + # allows it to be safely pickled + raise AttributeError(name) + return _RemoteMethod(self._pyroInvoke, name) + + def __repr__(self): + connected="connected" if self._pyroConnection else "not connected" + return "<%s.%s at 0x%x, %s, for %s>" % (self.__class__.__module__, self.__class__.__name__, + id(self), connected, self._pyroUri) + + def __unicode__(self): + return str(self) + + def __getstate__(self): + return self._pyroUri, self._pyroOneway, self.__pyroTimeout # skip the connection + + def __getstate_for_dict__(self): + return self._pyroUri.asString(), tuple(self._pyroOneway), self.__pyroTimeout + + def __setstate__(self, state): + self._pyroUri, self._pyroOneway, self.__pyroTimeout = state + self._pyroConnection=None + self._pyroSeq=0 + self.__pyroLock=threadutil.Lock() + self.__pyroConnLock=threadutil.Lock() + + def __copy__(self): + uriCopy=URI(self._pyroUri) + return Proxy(uriCopy) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self._pyroRelease() + + def __eq__(self, other): + if other is self: + return True + return isinstance(other, Proxy) and other._pyroUri == self._pyroUri and other._pyroOneway == self._pyroOneway + + def __ne__(self, other): + if other and isinstance(other, Proxy): + return other._pyroUri != self._pyroUri or other._pyroOneway != self._pyroOneway + return True + + def __hash__(self): + return hash(self._pyroUri) ^ hash(frozenset(self._pyroOneway)) + + def _pyroRelease(self): + """release the connection to the pyro daemon""" + with self.__pyroConnLock: + if self._pyroConnection is not None: + self._pyroConnection.close() + self._pyroConnection=None + log.debug("connection released") + + def _pyroBind(self): + """ + Bind this proxy to the exact object from the uri. That means that the proxy's uri + will be updated with a direct PYRO uri, if it isn't one yet. + If the proxy is already bound, it will not bind again. + """ + return self.__pyroCreateConnection(True) + + def __pyroGetTimeout(self): + return self.__pyroTimeout + + def __pyroSetTimeout(self, timeout): + self.__pyroTimeout=timeout + if self._pyroConnection is not None: + self._pyroConnection.timeout=timeout + _pyroTimeout=property(__pyroGetTimeout, __pyroSetTimeout) + + def _pyroInvoke(self, methodname, vargs, kwargs, flags=0): + """perform the remote method call communication""" + if self._pyroConnection is None: + # rebind here, don't do it from inside the invoke because deadlock will occur + self.__pyroCreateConnection() + serializer = util.get_serializer(Pyro4.config.SERIALIZER) + data, compressed = serializer.serializeCall( + self._pyroConnection.objectId, methodname, vargs, kwargs, + compress=Pyro4.config.COMPRESSION) + if compressed: + flags |= Pyro4.message.FLAGS_COMPRESSED + if methodname in self._pyroOneway: + flags |= Pyro4.message.FLAGS_ONEWAY + with self.__pyroLock: + self._pyroSeq=(self._pyroSeq+1)&0xffff + if Pyro4.config.LOGWIRE: + log.debug("proxy wiredata sending: msgtype=%d flags=0x%x ser=%d seq=%d data=%r" % (Pyro4.message.MSG_INVOKE, flags, serializer.serializer_id, self._pyroSeq, data)) + msg = Message(Pyro4.message.MSG_INVOKE, data, serializer.serializer_id, flags, self._pyroSeq) + try: + self._pyroConnection.send(msg.to_bytes()) + del msg # invite GC to collect the object, don't wait for out-of-scope + if flags & Pyro4.message.FLAGS_ONEWAY: + return None # oneway call, no response data + else: + msg = Message.recv(self._pyroConnection, [Pyro4.message.MSG_RESULT]) + if Pyro4.config.LOGWIRE: + log.debug("proxy wiredata received: msgtype=%d flags=0x%x ser=%d seq=%d data=%r" % (msg.type, msg.flags, msg.serializer_id, msg.seq, msg.data) ) + self.__pyroCheckSequence(msg.seq) + if msg.serializer_id != serializer.serializer_id: + error = "invalid serializer in response: %d" % msg.serializer_id + log.error(error) + raise errors.ProtocolError(error) + data = serializer.deserializeData(msg.data, compressed=msg.flags & Pyro4.message.FLAGS_COMPRESSED) + if msg.flags & Pyro4.message.FLAGS_EXCEPTION: + if sys.platform=="cli": + util.fixIronPythonExceptionForPickle(data, False) + raise data + else: + return data + except (errors.CommunicationError, KeyboardInterrupt): + # Communication error during read. To avoid corrupt transfers, we close the connection. + # Otherwise we might receive the previous reply as a result of a new methodcall! + # Special case for keyboardinterrupt: people pressing ^C to abort the client + # may be catching the keyboardinterrupt in their code. We should probably be on the + # safe side and release the proxy connection in this case too, because they might + # be reusing the proxy object after catching the exception... + self._pyroRelease() + raise + + def __pyroCheckSequence(self, seq): + if seq!=self._pyroSeq: + err="invoke: reply sequence out of sync, got %d expected %d" % (seq, self._pyroSeq) + log.error(err) + raise errors.ProtocolError(err) + + def __pyroCreateConnection(self, replaceUri=False): + """ + Connects this proxy to the remote Pyro daemon. Does connection handshake. + Returns true if a new connection was made, false if an existing one was already present. + """ + with self.__pyroConnLock: + if self._pyroConnection is not None: + return False # already connected + from Pyro4.naming import resolve # don't import this globally because of cyclic dependancy + uri=resolve(self._pyroUri) + # socket connection (normal or Unix domain socket) + conn=None + log.debug("connecting to %s", uri) + connect_location=uri.sockname if uri.sockname else (uri.host, uri.port) + with self.__pyroLock: + try: + if self._pyroConnection is not None: + return False # already connected + sock=socketutil.createSocket(connect=connect_location, reuseaddr=Pyro4.config.SOCK_REUSE, timeout=self.__pyroTimeout) + conn=socketutil.SocketConnection(sock, uri.object) + # Do handshake. For now, no need to send anything. (message type CONNECT is not yet used) + msg = Message.recv(conn, None) + # any trailing data (dataLen>0) is an error message, if any + except Exception: + x=sys.exc_info()[1] + if conn: + conn.close() + err="cannot connect: %s" % x + log.error(err) + if isinstance(x, errors.CommunicationError): + raise + else: + ce = errors.CommunicationError(err) + ce.__cause__ = x + raise ce + else: + if msg.type==Pyro4.message.MSG_CONNECTFAIL: + error="connection rejected" + if msg.data: + data = msg.data + if sys.version_info>=(3, 0): + data=str(msg.data, "utf-8") + error+=", reason: " + data + conn.close() + log.error(error) + raise errors.CommunicationError(error) + elif msg.type==Pyro4.message.MSG_CONNECTOK: + self._pyroConnection=conn + if replaceUri: + self._pyroUri=uri + log.debug("connected to %s", self._pyroUri) + return True + else: + conn.close() + err="connect: invalid msg type %d received" % msg.type + log.error(err) + raise errors.ProtocolError(err) + + def _pyroReconnect(self, tries=100000000): + """(re)connect the proxy to the daemon containing the pyro object which the proxy is for""" + self._pyroRelease() + while tries: + try: + self.__pyroCreateConnection() + return + except errors.CommunicationError: + tries-=1 + if tries: + time.sleep(2) + msg="failed to reconnect" + log.error(msg) + raise errors.ConnectionClosedError(msg) + + def _pyroBatch(self): + """returns a helper class that lets you create batched method calls on the proxy""" + return _BatchProxyAdapter(self) + + def _pyroAsync(self): + """returns a helper class that lets you do asynchronous method calls on the proxy""" + return _AsyncProxyAdapter(self) + + def _pyroInvokeBatch(self, calls, oneway=False): + flags=Pyro4.message.FLAGS_BATCH + if oneway: + flags|=Pyro4.message.FLAGS_ONEWAY + return self._pyroInvoke("", calls, None, flags) + + +class _BatchedRemoteMethod(object): + """method call abstraction that is used with batched calls""" + def __init__(self, calls, name): + self.__calls = calls + self.__name = name + + def __getattr__(self, name): + return _BatchedRemoteMethod(self.__calls, "%s.%s" % (self.__name, name)) + + def __call__(self, *args, **kwargs): + self.__calls.append((self.__name, args, kwargs)) + + +class _BatchProxyAdapter(object): + """Helper class that lets you batch multiple method calls into one. + It is constructed with a reference to the normal proxy that will + carry out the batched calls. Call methods on this object thatyou want to batch, + and finally call the batch proxy itself. That call will return a generator + for the results of every method call in the batch (in sequence).""" + def __init__(self, proxy): + self.__proxy=proxy + self.__calls=[] + + def __getattr__(self, name): + return _BatchedRemoteMethod(self.__calls, name) + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + def __copy__(self): + return self + + def __resultsgenerator(self, results): + for result in results: + if isinstance(result, futures._ExceptionWrapper): + result.raiseIt() # re-raise the remote exception locally. + else: + yield result # it is a regular result object, yield that and continue. + + def __call__(self, oneway=False, async=False): + if oneway and async: + raise errors.PyroError("async oneway calls make no sense") + if async: + return _AsyncRemoteMethod(self, "")() + else: + results=self.__proxy._pyroInvokeBatch(self.__calls, oneway) + self.__calls=[] # clear for re-use + if not oneway: + return self.__resultsgenerator(results) + + def _pyroInvoke(self, name, args, kwargs): + # ignore all parameters, we just need to execute the batch + results=self.__proxy._pyroInvokeBatch(self.__calls) + self.__calls=[] # clear for re-use + return self.__resultsgenerator(results) + + +class _AsyncProxyAdapter(object): + def __init__(self, proxy): + self.__proxy=proxy + + def __getattr__(self, name): + return _AsyncRemoteMethod(self.__proxy, name) + + +class _AsyncRemoteMethod(object): + """async method call abstraction (call will run in a background thread)""" + def __init__(self, proxy, name): + self.__proxy = proxy + self.__name = name + + def __getattr__(self, name): + return _AsyncRemoteMethod(self.__proxy, "%s.%s" % (self.__name, name)) + + def __call__(self, *args, **kwargs): + result=futures.FutureResult() + thread=threadutil.Thread(target=self.__asynccall, args=(result, args, kwargs)) + thread.setDaemon(True) + thread.start() + return result + + def __asynccall(self, asyncresult, args, kwargs): + try: + # use a copy of the proxy otherwise calls would be serialized, + # and use contextmanager to close the proxy after we're done + with self.__proxy.__copy__() as proxy: + value = proxy._pyroInvoke(self.__name, args, kwargs) + asyncresult.value=value + except Exception: + # ignore any exceptions here, return them as part of the async result instead + asyncresult.value=futures._ExceptionWrapper(sys.exc_info()[1]) + + +def batch(proxy): + """convenience method to get a batch proxy adapter""" + return proxy._pyroBatch() + + +def async(proxy): + """convenience method to get an async proxy adapter""" + return proxy._pyroAsync() + + +def pyroObjectToAutoProxy(self): + """reduce function that automatically replaces Pyro objects by a Proxy""" + if Pyro4.config.AUTOPROXY: + daemon = getattr(self, "_pyroDaemon", None) + if daemon: + # only return a proxy if the object is a registered pyro object + return Pyro4.core.Proxy(daemon.uriFor(self)) + return self + + +class DaemonObject(object): + """The part of the daemon that is exposed as a Pyro object.""" + def __init__(self, daemon): + self.daemon=daemon + + def registered(self): + """returns a list of all object names registered in this daemon""" + return list(self.daemon.objectsById.keys()) + + def ping(self): + """a simple do-nothing method for testing purposes""" + pass + + +class Daemon(object): + """ + Pyro daemon. Contains server side logic and dispatches incoming remote method calls + to the appropriate objects. + """ + def __init__(self, host=None, port=0, unixsocket=None, nathost=None, natport=None): + _check_hmac() # check if hmac secret key is set + if host is None: + host=Pyro4.config.HOST + if nathost is None: + nathost=Pyro4.config.NATHOST + if natport is None: + natport=Pyro4.config.NATPORT or None + if nathost and unixsocket: + raise ValueError("cannot use nathost together with unixsocket") + if (nathost is None) ^ (natport is None): + raise ValueError("must provide natport with nathost") + if Pyro4.config.SERVERTYPE=="thread": + self.transportServer=SocketServer_Threadpool() + elif Pyro4.config.SERVERTYPE=="multiplex": + # choose the 'best' multiplexing implementation + if os.name=="java": + raise NotImplementedError("select or poll-based server is not supported for jython, use thread server instead") + self.transportServer = SocketServer_Poll() if socketutil.hasPoll else SocketServer_Select() + else: + raise errors.PyroError("invalid server type '%s'" % Pyro4.config.SERVERTYPE) + self.transportServer.init(self, host, port, unixsocket) + #: The location (str of the form ``host:portnumber``) on which the Daemon is listening + self.locationStr=self.transportServer.locationStr + log.debug("created daemon on %s", self.locationStr) + natport_for_loc = natport + if natport==0: + # expose internal port number as NAT port as well. (don't use port because it could be 0 and will be chosen by the OS) + natport_for_loc = int(self.locationStr.split(":")[1]) + #: The NAT-location (str of the form ``nathost:natportnumber``) on which the Daemon is exposed for use with NAT-routing + self.natLocationStr = "%s:%d" % (nathost, natport_for_loc) if nathost else None + if self.natLocationStr: + log.debug("NAT address is %s", self.natLocationStr) + pyroObject=DaemonObject(self) + pyroObject._pyroId=constants.DAEMON_NAME + #: Dictionary from Pyro object id to the actual Pyro object registered by this id + self.objectsById={pyroObject._pyroId: pyroObject} + self.__mustshutdown=threadutil.Event() + self.__loopstopped=threadutil.Event() + self.__loopstopped.set() + # assert that the configured serializers are available, and remember their ids: + self.__serializer_ids = set([util.get_serializer(ser_name).serializer_id for ser_name in Pyro4.config.SERIALIZERS_ACCEPTED]) + log.debug("accepted serializers: %s" % Pyro4.config.SERIALIZERS_ACCEPTED) + + @property + def sock(self): + return self.transportServer.sock + + @property + def sockets(self): + return self.transportServer.sockets + + @staticmethod + def serveSimple(objects, host=None, port=0, daemon=None, ns=True, verbose=True): + """ + Very basic method to fire up a daemon (or supply one yourself). + objects is a dict containing objects to register as keys, and + their names (or None) as values. If ns is true they will be registered + in the naming server as well, otherwise they just stay local. + """ + if not daemon: + daemon=Daemon(host, port) + with daemon: + if ns: + ns=Pyro4.naming.locateNS() + for obj, name in objects.items(): + if ns: + localname=None # name is used for the name server + else: + localname=name # no name server, use name in daemon + uri=daemon.register(obj, localname) + if verbose: + print("Object {0}:\n uri = {1}".format(repr(obj), uri)) + if name and ns: + ns.register(name, uri) + if verbose: + print(" name = {0}".format(name)) + if verbose: + print("Pyro daemon running.") + daemon.requestLoop() + + def requestLoop(self, loopCondition=lambda: True): + """ + Goes in a loop to service incoming requests, until someone breaks this + or calls shutdown from another thread. + """ + self.__mustshutdown.clear() + log.info("daemon %s entering requestloop", self.locationStr) + try: + self.__loopstopped.clear() + condition=lambda: not self.__mustshutdown.isSet() and loopCondition() + self.transportServer.loop(loopCondition=condition) + finally: + self.__loopstopped.set() + log.debug("daemon exits requestloop") + + def events(self, eventsockets): + """for use in an external event loop: handle any requests that are pending for this daemon""" + return self.transportServer.events(eventsockets) + + def shutdown(self): + """Cleanly terminate a daemon that is running in the requestloop. It must be running + in a different thread, or this method will deadlock.""" + log.debug("daemon shutting down") + self.__mustshutdown.set() + self.transportServer.wakeup() + time.sleep(0.05) + self.close() + self.__loopstopped.wait() + log.info("daemon %s shut down", self.locationStr) + + def _handshake(self, conn): + """Perform connection handshake with new clients""" + # For now, client is not sending anything. Just respond with a CONNECT_OK. + # We need a minimal amount of data or the socket will remain blocked + # on some systems... (messages smaller than 40 bytes) + # Return True for successful handshake, False if something was wrong. + # We default to the marshal serializer to send message payload of "ok" + ser = util.get_serializer("marshal") + data = ser.dumps("ok") + msg = Message(Pyro4.message.MSG_CONNECTOK, data, ser.serializer_id, 0, 1) + conn.send(msg.to_bytes()) + return True + + def handleRequest(self, conn): + """ + Handle incoming Pyro request. Catches any exception that may occur and + wraps it in a reply to the calling side, as to not make this server side loop + terminate due to exceptions caused by remote invocations. + """ + request_flags=0 + request_seq=0 + request_serializer_id = util.MarshalSerializer.serializer_id + wasBatched=False + isCallback=False + try: + msg = Message.recv(conn, [Pyro4.message.MSG_INVOKE, Pyro4.message.MSG_PING]) + request_flags = msg.flags + request_seq = msg.seq + request_serializer_id = msg.serializer_id + if Pyro4.config.LOGWIRE: + log.debug("daemon wiredata received: msgtype=%d flags=0x%x ser=%d seq=%d data=%r" % (msg.type, msg.flags, msg.serializer_id, msg.seq, msg.data) ) + if msg.type == Pyro4.message.MSG_PING: + # return same seq, but ignore any data (it's a ping, not an echo). Nothing is deserialized. + msg = Message(Pyro4.message.MSG_PING, b"pong", msg.serializer_id, 0, msg.seq) + if Pyro4.config.LOGWIRE: + log.debug("daemon wiredata sending: msgtype=%d flags=0x%x ser=%d seq=%d data=%r" % (msg.type, msg.flags, msg.serializer_id, msg.seq, msg.data)) + conn.send(msg.to_bytes()) + return + if msg.serializer_id not in self.__serializer_ids: + raise errors.ProtocolError("message used serializer that is not accepted: %d" % msg.serializer_id) + serializer = util.get_serializer_by_id(msg.serializer_id) + objId, method, vargs, kwargs = serializer.deserializeCall(msg.data, compressed=msg.flags & Pyro4.message.FLAGS_COMPRESSED) + del msg # invite GC to collect the object, don't wait for out-of-scope + obj = self.objectsById.get(objId) + if obj is not None: + if kwargs and sys.version_info<(2, 6, 5) and os.name!="java": + # Python before 2.6.5 doesn't accept unicode keyword arguments + kwargs = dict((str(k), kwargs[k]) for k in kwargs) + if request_flags & Pyro4.message.FLAGS_BATCH: + # batched method calls, loop over them all and collect all results + data=[] + for method, vargs, kwargs in vargs: + method=util.resolveDottedAttribute(obj, method, Pyro4.config.DOTTEDNAMES) + try: + result=method(*vargs, **kwargs) # this is the actual method call to the Pyro object + except Exception: + xt, xv = sys.exc_info()[0:2] + log.debug("Exception occurred while handling batched request: %s", xv) + xv._pyroTraceback=util.formatTraceback(detailed=Pyro4.config.DETAILED_TRACEBACK) + if sys.platform=="cli": + util.fixIronPythonExceptionForPickle(xv, True) # piggyback attributes + data.append(futures._ExceptionWrapper(xv)) + break # stop processing the rest of the batch + else: + data.append(result) + wasBatched=True + else: + # normal single method call + method=util.resolveDottedAttribute(obj, method, Pyro4.config.DOTTEDNAMES) + if request_flags & Pyro4.message.FLAGS_ONEWAY and Pyro4.config.ONEWAY_THREADED: + # oneway call to be run inside its own thread + thread=threadutil.Thread(target=method, args=vargs, kwargs=kwargs) + thread.setDaemon(True) + thread.start() + else: + isCallback=getattr(method, "_pyroCallback", False) + data=method(*vargs, **kwargs) # this is the actual method call to the Pyro object + else: + log.debug("unknown object requested: %s", objId) + raise errors.DaemonError("unknown object") + if request_flags & Pyro4.message.FLAGS_ONEWAY: + return # oneway call, don't send a response + else: + data, compressed = serializer.serializeData(data, compress=Pyro4.config.COMPRESSION) + response_flags=0 + if compressed: + response_flags |= Pyro4.message.FLAGS_COMPRESSED + if wasBatched: + response_flags |= Pyro4.message.FLAGS_BATCH + if Pyro4.config.LOGWIRE: + log.debug("daemon wiredata sending: msgtype=%d flags=0x%x ser=%d seq=%d data=%r" % (Pyro4.message.MSG_RESULT, response_flags, serializer.serializer_id, request_seq, data)) + msg = Message(Pyro4.message.MSG_RESULT, data, serializer.serializer_id, response_flags, request_seq) + conn.send(msg.to_bytes()) + except Exception: + xt, xv = sys.exc_info()[0:2] + if xt is not errors.ConnectionClosedError: + log.debug("Exception occurred while handling request: %r", xv) + if not request_flags & Pyro4.message.FLAGS_ONEWAY: + # only return the error to the client if it wasn't a oneway call + tblines=util.formatTraceback(detailed=Pyro4.config.DETAILED_TRACEBACK) + self._sendExceptionResponse(conn, request_seq, request_serializer_id, xv, tblines) + if isCallback or isinstance(xv, (errors.CommunicationError, errors.SecurityError)): + raise # re-raise if flagged as callback, communication or security error. + + def _sendExceptionResponse(self, connection, seq, serializer_id, exc_value, tbinfo): + """send an exception back including the local traceback info""" + exc_value._pyroTraceback=tbinfo + if sys.platform=="cli": + util.fixIronPythonExceptionForPickle(exc_value, True) # piggyback attributes + serializer = util.get_serializer_by_id(serializer_id) + try: + data, compressed = serializer.serializeData(exc_value) + except: + # the exception object couldn't be serialized, use a generic PyroError instead + xt, xv, tb = sys.exc_info() + msg = "Error serializing exception: %s. Original exception: %s: %s" % (str(xv), type(exc_value), str(exc_value)) + exc_value = errors.PyroError(msg) + exc_value._pyroTraceback=tbinfo + if sys.platform=="cli": + util.fixIronPythonExceptionForPickle(exc_value, True) # piggyback attributes + data, compressed = serializer.serializeData(exc_value) + flags = Pyro4.message.FLAGS_EXCEPTION + if compressed: + flags |= Pyro4.message.FLAGS_COMPRESSED + if Pyro4.config.LOGWIRE: + log.debug("daemon wiredata sending (error response): msgtype=%d flags=0x%x ser=%d seq=%d data=%r" % (Pyro4.message.MSG_RESULT, flags, serializer.serializer_id, seq, data)) + msg = Message(Pyro4.message.MSG_RESULT, data, serializer.serializer_id, flags, seq) + connection.send(msg.to_bytes()) + + def register(self, obj, objectId=None): + """ + Register a Pyro object under the given id. Note that this object is now only + known inside this daemon, it is not automatically available in a name server. + This method returns a URI for the registered object. + """ + if objectId: + if not isinstance(objectId, basestring): + raise TypeError("objectId must be a string or None") + else: + objectId="obj_"+uuid.uuid4().hex # generate a new objectId + if hasattr(obj, "_pyroId") and obj._pyroId != "": # check for empty string is needed for Cython + raise errors.DaemonError("object already has a Pyro id") + if objectId in self.objectsById: + raise errors.DaemonError("object already registered with that id") + # set some pyro attributes + obj._pyroId=objectId + obj._pyroDaemon=self + if Pyro4.config.AUTOPROXY: + # register a custom serializer for the type to automatically return proxies + # we need to do this for all known serializers + for ser in util._serializers.values(): + ser.register_type_replacement(type(obj), pyroObjectToAutoProxy) + # register the object in the mapping + self.objectsById[obj._pyroId]=obj + return self.uriFor(objectId) + + def unregister(self, objectOrId): + """ + Remove an object from the known objects inside this daemon. + You can unregister an object directly or with its id. + """ + if objectOrId is None: + raise ValueError("object or objectid argument expected") + if not isinstance(objectOrId, basestring): + objectId=getattr(objectOrId, "_pyroId", None) + if objectId is None: + raise errors.DaemonError("object isn't registered") + else: + objectId=objectOrId + objectOrId=None + if objectId==constants.DAEMON_NAME: + return + if objectId in self.objectsById: + del self.objectsById[objectId] + if objectOrId is not None: + del objectOrId._pyroId + del objectOrId._pyroDaemon + # Don't remove the custom type serializer because there may be + # other registered objects of the same type still depending on it. + + def uriFor(self, objectOrId=None, nat=True): + """ + Get a URI for the given object (or object id) from this daemon. + Only a daemon can hand out proper uris because the access location is + contained in them. + Note that unregistered objects cannot be given an uri, but unregistered + object names can (it's just a string we're creating in that case). + If nat is set to False, the configured NAT address (if any) is ignored and it will + return an URI for the internal address. + """ + if not isinstance(objectOrId, basestring): + objectOrId=getattr(objectOrId, "_pyroId", None) + if objectOrId is None: + raise errors.DaemonError("object isn't registered") + if nat: + loc=self.natLocationStr or self.locationStr + else: + loc=self.locationStr + return URI("PYRO:%s@%s" % (objectOrId, loc)) + + def close(self): + """Close down the server and release resources""" + log.debug("daemon closing") + if self.transportServer: + self.transportServer.close() + self.transportServer=None + + def __repr__(self): + return "<%s.%s at 0x%x, %s, %d objects>" % (self.__class__.__module__, self.__class__.__name__, + id(self), self.locationStr, len(self.objectsById)) + + def __enter__(self): + if not self.transportServer: + raise errors.PyroError("cannot reuse this object") + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + def __getstate__(self): + return {} # a little hack to make it possible to serialize Pyro objects, because they can reference a daemon + + def __getstate_for_dict__(self): + return self.__getstate__() + + +# decorators + +def callback(object): + """ + decorator to mark a method to be a 'callback'. This will make Pyro + raise any errors also on the callback side, and not only on the side + that does the callback call. + """ + object._pyroCallback=True + return object + + +try: + import serpent + def pyro_class_serpent_serializer(obj, serializer, stream, level): + # Override the default way that a Pyro URI/proxy/daemon is serialized. + # Because it defines a __getstate__ it would otherwise just become a tuple, + # and not be deserialized as a class. + d = Pyro4.util.SerializerBase.class_to_dict(obj) + serializer.ser_builtins_dict(d, stream, level) + serpent.register_class(URI, pyro_class_serpent_serializer) + serpent.register_class(Proxy, pyro_class_serpent_serializer) + serpent.register_class(Daemon, pyro_class_serpent_serializer) + serpent.register_class(futures._ExceptionWrapper, pyro_class_serpent_serializer) +except ImportError: + pass + + +def serialize_core_object_to_dict(obj): + return { + "__class__": "Pyro4.core." + obj.__class__.__name__, + "state": obj.__getstate_for_dict__() + } + +Pyro4.util.SerializerBase.register_class_to_dict(URI, serialize_core_object_to_dict) +Pyro4.util.SerializerBase.register_class_to_dict(Proxy, serialize_core_object_to_dict) +Pyro4.util.SerializerBase.register_class_to_dict(Daemon, serialize_core_object_to_dict) +Pyro4.util.SerializerBase.register_class_to_dict(futures._ExceptionWrapper, futures._ExceptionWrapper.__serialized_dict__) diff --git a/contrib/site-packages/Pyro4/core.pyo b/contrib/site-packages/Pyro4/core.pyo new file mode 100644 index 0000000000000000000000000000000000000000..07b3d9522deb0af5fab313ba9526ad4b6fb9a643 GIT binary patch literal 45729 zcmd6wd5~RKUf<8_y<4lbv@BbepJcVwvZa={8P90EXlu(FTiu%5vSqiVd478DcK2(& zey{cJYfEh`q8XW47-|d@3`+)vEg@6DPze+^r?q28!xu+3)nZ=bn4+S$_NPch2qp`nsVHfAxohii`jC@%Qj`=c2E5IahUV znzZ1mU9Mhm3teuy%N2Cp?dsiby4&uN?{U*TNxs)j_a^y1H{F-y``vVZk{@u>1D5Y` z^+C5V;3zBU44UF*yyG=`s+Sdzr!tTa?_jK z!e%$U*_ZZPYg^p(7Joh9>Ra8yu$vzC*YtGIO%J*%{|~w98dqKGs_R^Jy_=<{+g#Y= z=I?aX4X(P;&2+g-y{@&s5N>zVce#dhVZWQd+g0yyHwtcghZ+g*aWhQDCijMOH=KKU zdZ(*yPD;0=rT4n(R$WXN_1gHM;q>+{SLM~dsXNos-LASlDZMK#z0XzePD*#ArF&fU zo}_eVS~}vY_v&K$epfA8X0NO6vdpNf?oJ-KFMVL2rX=pjp0xA#i+eBGqZ1OXz28Et64lyJiXkWYc`4j zb=qNZ^5T40X-8wl;<4u9a;r8w*DhXME}m>HgpFb~EPk-rm@SUhba}W{ey!Phwbls7 z8ey9jYk$qZ(E@#FuP3=uYtNOUcDWr=xjix3s~dwPm1cwDMmuT`sc5bhma9wcTD{%F zrDaqeH7l3Gwp|LZ!&a-=ihS$axSq1xQ(^1XuvI!^Z7epM^-(b2wKv~fVJfUMto9u@ zm!EFd>%Pr?Zp|#Ums(-`5*3^Zqo_O^w!6u{5S(ml2&XlRZFT9m&1-vrY^7YUUo2NH zS&fV3c4f|T`1_T`$<@-^VcM4!tPXqh_&O z+AqYiY_hoz_nQcb8dGI%~@T`9(cpEz=AJe~kOc`luU z3v{kH+FVp4%&;}<8vy5_A5W%CFXgp%f0U0e7Ad@%L4ow`V`mg zQQhc+4tjv>!Of2r#E+@b!y~1B|7SqYmj&clK5w1L(V=3 zfpua-l`zn|7#Q4z6d5n?G^}Ytn3V39T#V^cTYfo0`Z<;i;}G* z)39eiZ$@q_i+#e<6`Qs^pzq#Rg;F~UXlB&_eC zSl<{`#5YT&TBFu3l^!4;DRG4jg|&rsg&zKHC~PZiD)jNYlb^1Ktz0FWyDGv=4Nk1s zF!W(C6ug1U3gU#ECjSG3tuwj-`rIodF~pcJxV6#KLLk9;!CmTetpkO2m$2Al50HEL zK%w!QhUTg_wfsiAWPEzE@KQ0_WVJTd^p?x_wsRTqGKK?p*XJe-dn~Ex!8RV!2v{-NU`Z zMy1}2p!zV9cC%SrC^ujSE#LvHigxha`E#S^9y;*;@`0Hn2c9`Ja^b;!7e)__p0A2_ zM|b5K7Ex9+KroUl7f+4~B@C5;=2knByy>w|m=x^NUG=aP8lW4J7=A@L^b}bFtJ*a& zQA3ehqiv18SgtR5+z#$hQvx?Ie^e6`2w@C!m~=oRoGCBW+o$Q3%}l8i_!R7+Cr9~3 zyGa~)BOF>+*k0J$CEVFmc%W-%VG!)<29GuuDq3XyG)IO%>&OVO60DgrVLCcQaaFW> z0a0I4Ksgg5JSmjaBl)IWyj4_6A{sJr z4d+Ip3x`Ikkx*nLQZB*#2@lyR=klnulX`x=1k)XXYXQo6ev$Cqbq@l|Te`LZ$!of7 zrL;=vO1+N@zd9__Hs$zp&97LO$!RRWVN|pk;bU$jzPN>E<(#t zSxoJakRrt-7F%py4doBp!x_K!Y*h^!tuW$4J5Y(vGTd@BWzZvi)YMVewS}qdE8O7; zf{1}2!odCr89$jfka11$1!jTROHkD(MH0o~DDQEX22{eOIG^XBUB-i_46r9MCD`PT za#X3+Ojcoj>*Y#lq$aCm4+Ww;p6)J3CXPv$gia$g%rs5nlBj6SplDW_P$^*%T!*Z+ zm`%Q4&Q1WZ^{<2_q*W?irhCxD}68g+c4@A=zY$DpsV^54kVucdvK_PZG_!8$`lRWJ-xN(EA@fTouK zk~$M?S7J(!*uXOwEW#Y+(gI`>QVxw&xd~}(jgnb)bwSfqDutIzrEBD3B-((Gv#Vgr zdNQ4cL}C>l-k1a^)Y@eHLbKp`l5AYTbGqc$p&>N@SLkaB7wV#ZDO{*;f(v72N>ghC z`2;3_1IC#QQ_mWKoGV9jZ@soQ8*08I!yn+qrILDBDt(N6f*8pJ2;OWpcaw=%^DX9p z(I-RKQ}5Cd2Y1o8Z{v6KNHyYjApUx^!-P!sjGTo8iN92>RnXqw%nd0)!xhjA5}r4* zg&@sBdJVkfwOS)>C9K?1Ml})MQ}EVHs5cm24;~=b3CIoZ=EdL1FJ4D4y9=BXJ5!|L zd()Ots>IM}(zXnZmXR8pt2rvaMsPyOGbFLxP;FL9rQl#vfK9xx;z+|^&dgbZpp?0Jd@>7b*yk0MsFCx~JD>5FK zJ*u1gf=@$cDK0?10G^D8_ShQg;aoM%SGA&}ENIxM-mYAtjn^|Y`$pfbs=8+P_`BLn z%%JXjd9OjVLzhUoaZ10OC&jlJ*`WA zt3B3O{KZCGZcCc~EM3Txa;?7BV)8oWw)(KEWZZ%T?Dz83W~-kYo*p)`X6ssR%|@oo zT)LucM@Bn9j}W$j^HS+^d3LubK@@;7nUNt=pl~ub{Ls^!+<%b?uw8HBTDX9&ouMN)4`J}eoD#H zx+f&DhRlaxX3maHw?~M|Tv)kOnp-GWew4l$Lz8;Dw~)@e#$zP+K|nvukz5?y&;#hL zw;5nsz%hl(Y1Yi?ip%jnZpzKUt%w|RZ7Uqd`?XmQ@7KnV?)}=>|Gi%uKe+d6;{^A9 zZG33nuZ@e$B46YL_kL|$UEZ&a6Wm9<-~_io7dgSV86&_6Zb2?`f^WByVHXnrvekvd za(+z{fOD6kT;9MDzui?W%7p-&kkFfZtdP(f;%o?Qp~5{$XbUAiwB=r_x!dj*#dPk| zgsx@M%&jW_NgT!D&8;mTwBZGfvEoU&Q!C+OJ1XXw6nb={TCCQhMR}eCfOFx3{IIfe zt!%-2vd6}X=D;fT%5KB8C( zyKT`WCWtmdqmHQi8<;37l0QilGbvNwkNH>2!Y@d)@w|X1Y)x`8sfL!l$4?KOsV#)f zrFK?(D!goLO$0;~-iXN8$UsDHpsry9gubJ#0|c#+%rYk9;!gQfbQiLLEBKfgH$V?v zi9NZ-5K9L7k`?-e|1_#RFzw}rAxCc&Mr59x(t5{;@y^E8?pS%bDR$<@kKoe9C*2-) z^V@`H^85Zt(j0&c(X$edeDynHE9pG&OIPkvrwdp}LCmR9ObNR3ve|{U`(ezd>_w$wi?5He(xFn^`6`GlsZ&ioSzj!b5<6hw=ympp${RR8g(L zge|FM4d}2{2+%V^5!1a6E9U)lwN$Ex$Z20B8%yFFGD$og`53oFifpv)~es*mkd`csb4C%N&5pyMBmOgQlJfcg?3QMsV0WRWC?yB#%ZewDs>w=gkRVx~E=sBRnnN(l1! zD`+7~(Ij8bL=tp1BrqAtAVr)b8>8`30(lf--U%}0<@!>E*Oy9vq9F@36na(>8A^UO zSlwaalX0nDXR$dp`Vx^q;Co3rNCM~Ts($Lf^?f}V;j!0Is@W)v*k0vs7R_XH5?-y8 z%%wiaJzWrgxfI$PW#$YO;@-gp3$H3x@XzU6GUQuO`-p~Yn5&3{je^suymHn&tlM&z z8hu0aiA%g6&2*t;t&Os2LeI38LNEFYjx+4eAi85dL^IcaLo*@)L1vepOzczN4G=y+ zhGDlJH@pR1z1aJ>&4s7+gbjAP#rLk99he$YO@2OZHx0qZX#U+W7pLgQU-3(&kKGwU zr*l!HLY{h&&$Bz|*^y+xdM21qA|hsJFw<(j7B&b9Hg4BhN?;C}|3}XfY)A-~K*82s z;+Q7@Zys_5<;>V7W9HL#!%zX%R*20EfQAoVHNq}k=qz_W^b$!N9<`4qTi)7Z2BcVm z#kP`|lw?4O_Z(R;N3KTvV0FTav*LVSrj`0^mQ9NnH65BAOjc42FD}h?c%nrr3(RgD zF*O9ISSL?MGT|v}Tt`Je;X<)Qx{dyb;?%?;OPUWN!&}IPtk@D4Ud4};gu?@7nCP(2 z!6nDoAwwk0LRfCdc1Kn>?Ag7MDx@uI|Ap+#ON&)@T~)owqm1cKCXk{c<9+xnF`Vqy zsF{^SduEoyw%>mtgU(4sxW(=J?JU}#Mc8h0sZrgZZ1X5ed%38QlxK-i%pNy7(B8p- z^CpX9w9yV_qo60iiYFPGK680=gr|tdrKZh6t*(~Q^`)z-?im{OlkuyXPzLMy0>zyL zOm7;pL7WKNvGwSVbSN1oNC8xq6VD&zlk^0^nRjDvTn>pSb*Fv}4K$7n)o^pzs!5S; z2s>$x_J5kvj;y8`$!sR3(wW$znecp3l5CB>xr(^DG*Ahrwz_XJIevBVF-U6g)*x)T zAUzZiYJ@oiT0}^dB!2{6Vi~x!O*;rz^X%I}y--gqPOW3)rZA;qWy!Jun+Z_xhl9x4p~F?~X5Gvpdh- z7rz8Ux1QUP%*R@H_(q|?%B?8f!zf?uYLB>iHYE?3)%;YKpaazDbM4W@QqF~5!9-#* zKik;L)(qah&$S;&c4xeyO~M+^g9cI2td?0cEi&2|;YZIfImy8evpVDQw5=GZ$*7l&qrNx1y?>L4p+h)mo%uZI#G>Jw2 zbTB?Ob#ijTyB6YIzalBmJb&cGR4J)3e%!=9v;X=#lM~}-kG$*^`8mB$WbygQ6K78b z)wNn}spu>CuUxb^v6hI94#7gSJMCKY+t4Jm;9jsTpMEQvylGXU-ec#% zaW$zjKj!h5j*Xu_!)%#r+B!@OcUy4UzL#1vwW}xD$mjQojbD{R5|;l=vvs;wxrCkQ zgVX^zA-{R(^>3E@=ecnCVzb<;nme<_ez?F?RhuI57Ml!@1&VsH;k+uJUvPoD!RL}H zY5!Zn59`Kv=!O6VpbvhOznx%I0ch{;5@XUdxULOd>oAh`@l(1PHnq7hRM>(dcd)R> zw(4{fCf>u=owbEW$ZbOMTt_+**B$H{C=B-xc~-B%>1TXxoQ!An^Kf|R;}NrId5a#a zMC=XwR*_R>#D?3+M`*0hUwA56P0d@;dub1>#ejbZJYB@42FMve6_Ek7GeREP{l!@% zL#RgT#2cwvbcDBk8A@pGee}YQk;{3Neuzt3A)vDpi{l+~CPf^}W6X0}uK_wI8_{Ow zzaoK3cq6Q^+5YBKig>fxm8e=9O|rU2W17OA2+13Z~)sz$eus~6c7j? zA3neYQ3CHvfj>wWt*UL}9dzuiRSEn5NY`iyC5q6>TKDGH3x(y-E_jiD@n`JCP#eD^ z7zU4z*E^2qiE@Cw@#rR4P&cej)i#?PK-*jNcJ*hgn;#aVr$v?Yv(@k;@Uv~U$#Bj# z8Qv+5e*=Qi@3zC^soc6#yM^a*9eW8%TOkL4Hh8ifLz3`h1#8zxjIGOm{SAp9*Vs7Ct@~W-iv^##w$`=j;2y<& zD7NQc7U19h7GWbA(pruK>|G3L*Jc3wfU$+Y(2qIm+EBBjF@PRHy z&(zAMjL~IJ7ptYqJ~bS#@Oqnj7N13K?SioVn1|gT5HeOgj^{jaNlBrfBX9&K`=u~b?*fkGXh{#n zB)SB{#EL&ZoaA9p20kylTmk*w?xa*W@$4x6!HFHyrpP?|7ZDSOT6 zKAtoeKu)4D1}RgcR_C!Xm>g3V{8QBom85dTO`FZV7$U2cNhlRlx~B*Vz+9ys2KVch z_~_K+vFA#`_zMICNUbwH(pSvs=$bk&nEZh143ebWJY)Sb6Kg+K@Cj8n(k|)4)A!FO z4=Rq3#UV=hY2Eyc$~BP^ijGZAOpG5p^URTx&wEB~!ETSKib!m(%H(rqI0%ew=$=^* zY(()sGQxWNL}^}49@0|T_~JDyCHpAtt^%`zFQ~5M<*UI@D=Xs8u>dv}Ne&%hMe;NH zsF0hDCm#|wI?rv_D@#-lQo)9iS(F^ovv=#1)7dL0Rip^@CwuwUJ_tKh;GCq3Z%3Q57mwV7}W}fg0 zlqSH4#Y~x~dqa0n+1s(~Obe4uy$+&D3Jl^61+k5oXd2bSaM9zeAPSZ$$v3ft9kUJc zCj4fCkUd90RTvp;b_D*`(@m3vf@Hj(65QQMu>(d1fwZB*01!3=Jc~|I){5Ckxi}Zr(Q%8mrH@6xx>@IJ06S^M)32f(r^M9f^j;nHldYK9 z;7p+?Sqve%%+=*1)$$@9vJTtlG=7^z>)7{e0yQ_z?_7SzshaqkPF0l?ha`Q^p?5Wx zZVLZ=i@CCZ%6H>jS;u}bH&>!SLaCGr$)5#SL2uAqbf5+TAYvUcIKHqO+nL!gMU=cN zAEE)}g4W2Nr8Sta{qRp1bj9H{#v_e2$vEhh-J?g&9D6qSUan1ZGhcqrYpO0OLf!}K zBi#^3Y8q%qaMp~VqHR9V%7E@iG?4>zQ_ewYn4`Nim=j8#QSu=rA6N34k_VM6D^ZM; z-xG9Pxx18{R-#y?;JT8JDESs8&nppX`)xzg*n@8)xg}mKGgRMlE3@1NqU*t|5|+>f zKceJ=O3d2x@0I&iCBLTRKagbeZMjtpMwFl?svo_Nq;IgVu&Zlum~97rgT1yVse7tu{>oiW!LL)Xb8&*ghMy1yiT?b(?r!>u z#`POY6fl=_p-k(NpHEg1kcIC*t+5)5^4`sa^iE!;`^jXG^l+>w` z95uOJ=8ezTuw)67iG4kz?`J=3P+F4WpORxS3p;$^te#^T?`ZJnj`iMP|;lL__ztHd(1pYl_RuEx9 zy-kKBAA~;I=cigksUx5x+4Prh>bLr=p8?LhZ>q1;wO`3~S<@^yS5GU~ znddzByo_tR%e|<-XLIYL#(XTF$hjSiTuP05 zXIo;}Ewit-%C#sw+0NM}`gtIxVu8BSiFNhbektxoN@;&h(66XOoq4PMsBWV&(SS{lKPmOA-y&-iMIX`IAj`JEq z@SjynI#3`2*WN>Nt0AOx%VcF*Z1810ED{?0Hc2NI)V7U(Ma!pdNLAIzpBuXrh2B@F zX!)dS3m0q|#S*jXUZWTGWMAe_wXyjAq z>-F366YY4zn&68RSNPdK>UmLCI$|oyzJa95KJi4FMGzG*+$t8yEdX{he~3lZjESmM zFYg!z-9{S9i3Yz(t-qz05xMJktcT&)EQ%UQbb?`n$KPA5vYu#H3oxbrRU-eiUm zMs!YM7Nq=HvP>&90L`!=#yw)a#_a*jfL@?&zGF5`_%CLd57)cK zbZ~Qy`Rc_!h)aT{?~;AmBO?&2NQ=cc|56JfFOEc#JxG;AS{bRJWPqQ6-eoe&oPL_f zBP`3C_bPlwv+l!u%ts-yuv;NJlEgps67e{!5?c+=HV)t{rtKhwqe$$}=@pTHl$z0%UlWcpqf`6eKTF&6Nlsu{A=SjTXOY>{7 zkhWxAtj^mF{g&8hWBD< zDT6;G$p&D#)f;@P!BA3AaI7oL3eif-$lV9PA+`^bdIer9xidzlbX zP>XLa!ZX@k1>f^$iHGr4xzrCw+^l^vK|yl)rMis;`0Z4k;D7VJZ0h|bMXOD{&7XX4 ziVU~eGbcNCD!73UI(EPSJ*It6a6gL|sS`-3nUlReHsVWCjWDdwatoJ%|JsBC9%E3` z%o}v-S(^w0b^mm9beU}cs&Xk@j@|{~BH1{0r?ZxYi=;$TJ%(U-E!g-&Ir+X+wRu2i z7%AK`<`d?8;JY;{T3l)gz@iC{f<`Ho?FdF{6pQrg{%ojk?TGt7R1W@XMOUrUtH{f~ z!Q1aqUkAt-6UUREQigsCjWVdGD=5RT5qDDmKRP2{n(T2Q5!t3TLep`>>C%)(UG}?> zqwSIhamL9Xho8>Qd6rAgTy^r$QJL^%@1f(QJAY6fC*Apj@;K?vAC$)tHvXVIwvf>* zXXWuAVc`C(JUi@8ui(TyTz+gL^C#wUnu09ZaL=s#6>(;0O&&hS^%D`;)sCCq8gk+qBH zBX=5xYCf02J{3M)79+IxE7s_MtMrkGZE0nU^Im1~AT|&f3r7U&+3>eJR-v%&!g|G9 z(W3kDG%!+_Bsj`y#+K!z*|d=(tsk^X zX1(;4;`iAxlL~DAH;TNazcByOkZa5*Z>54ixou+sa>fn-fJ62FOi{A{NBUNdWt!jM zT4)t^UR;Ave?9}6Hn=_6Hi*LEhf1ivf0$RY*Ff6{scrhCG?*Ty&fl100z0_bALA$731M&D- z`TXU5_MZE+UqBFoDSZ}j@v}05w}rFu*u;3h?fA2=n-P!y;VyUi{ap-4&9lkR&jknc zC68$`eQ1Ov;fY9ENySH`6LZ67`#7N)p4M(6O>wwBUStlcV^g1fo`zV+UDUS|M1k~I zg)lr}UqiBr@#DcCQ_Ixs#OZ!D5DG+r68rjP5fAht-@80;k<;$9xzA7#HmTV7i0_SY z8a3tO74vpX-hRadfn_p2rPcA%b7^6qNt}D!lJ*f6_l`t+qb4q4@k;rh; zzL~x`UZ$yXMf=Bmf8+D`_^{MSH2MHH{h@q2mCMA` z9xk7qoH}C?Dvjpz`Yc-TC>4AZ{CM!-k^?$I(V}OYLg&O2fyUC3ou0=3>-t;{2-ym(+q*GCMSIs&wR-w*IjVvRf~a z-6QyiswbMCwhuhSE5**n$VW+wQD+~Ou-VT*+-5HZsy#!n;bl)D$gm~M1 z$P4YWA0`c0vWg;s(D@;W1kaFhg?{Wr`h=Z+TkY>Kd40ZalYNG3SJ(E!c5<6>Ypw0N zx9eUUJHyml+oipKoGa1wSSBNDt&MGEDl*{>i8o7Y9Q?JCUnJ?Q=xSX!uW3-Pa`DZ7 z!y(wRYcmD}ufrqROqXBDjYOnD^FPSX7USz=V1$sTcET5N62}3@-R{jV!ywrN$6-4b zpX~J+FFjoS9^)`LB~dOfe9|#vus@v-zyl?5bQlaA8jhu{bnL_<&;KwRYI4p1TiCuw zGKDt}a!ZT{`Jl_4cvf7LV+M{}W0vY^+P3=zJ^LkD7-UJXa|}*Ad-+T5@^SrtR(F3_ z(x`8TH=EQ^&~6O&mnmvmrYYY+I}5voVEh@ zthL~d(gyWTeU5XVs+t9qZ(5(6DYA79-eSAgsIF6C(IYuBC22X#nu~+6Xisuz%;ae8 zN?5P&FWMooSBQ;}7`j}X*Ad@*a$(s%(^k)h7HwzNCjBRozv-cC{?on~Mv6}szvhprj-%t_6~)>1<923E@x3!DXj~Dv>dhmbUUnaGU0$y~gk{p81S!d*xXeju;@GgAjiaWzzQSQrk01 zA&NRv*P!gA*iH~;8rxa{(7gSjLrL>o2 zAUnG}C1A$q6N_y8Nv@C4TsF8>W<9p*Pt?l`7pvu`KBD1ibmkEkdsctDuTy%J%qJij z2ET<5FXoHGOI zxQHlv$?s-Ou^vR1iwT`F7HNuhcv5nVvlm};_$X=78-|b~`9disBd=%LE7he6HSC`N zee-qtFYzWngKL|^?h~d(uHeOkz>3Gdh^-7BF4Tr?V^quNzg7FvXrP;`3@*Vgy7#RK zm<0E6HDMxlJR3S)PiC}q9z4BC2UX#iY%i}G>99SY-82#mDUlrFjckn5gh!KVv$!#P zp=l`4c0UW*4B1ValvR}6bP9u^jqLTVcO5cWx5bTZ>sp^hUT6W2T?YtMOB$@95$Z4? z?pHrL%N?mrQkMkS+V2cl>7(q#_!3#HwLfIrsR@SGH*n)w7AO@-)T_B6i197S$P)HNxs}fyK+5CKx_MzcK;k(|ReO+Cbc35n$f34W+_} zO*kSz%vH5sFz{qSv{1+ioVFH!vLmjRQX8XN)!K~3rlYi*a*Q+(FIsG-GO419o(vnj zG-ez2gC(-DP?9=TGGxIzfw(qiyS z3_JLDNZ0Z0t*troi2R z=S4Dk;DHyxe{dAQn!rH?K-l5iv8T+q6Cy6)0Z`m9yPe!;h|>U>3AH>`K#z8Ljf)Z)XcHU3$S><4{`5?r9&85jya7Mo+{tK$WZ?$pT)NQ`e7as)iIW3x$A=fWu zbr&Sl7HlgeD(=3{{VA3YJZGJhrkgTRU^r03;vbaux#i0}fGEqx@*tS9W-L;i8FaI( zE{}NY-H=*CX#wqNP#NDEPuVvyT0f?59?WCn_SHD*j5?ehVm1APO7B>4o)`rjpyuD~ z&6JN6=uuiC3S$j04&R2r2@7NT+Pe$!YcX0{Yf9hgwE!dA<*B$?)J(vL>2^v5K2~Il zyM!8dp@7wf!&fln`dhJ47A;TMz;SH$fG_~iCR7CbDkT{7lPQbGvE3cMQ7D{8wd8;l zCUT8~o_sTvXyq z@$_PYwF=9eO2d?{QTv~!iS@+Ge42Qf5sR1knvPP_i81$UGFhnVfZHXtdV{;h;(DOK zT5RXn6!e80Db?_hh)GjR)lZkh%Idx>WN)J=WfD3`XPOgq#DeJ?7>QT6;Fs z!s*Mqx?b|PUSDskNx@m3!X0q=Uv;31UX#{n z?#AxqM%>ss*LtNVMWy)ZLyndN*`NiOxGs;o8Jbdy<@L3w&ae$fMe=XLva^k!OTlKM z_wB&u^$m9N(Fg4KrairJ$3Qs`MJr|~;{NTR>YBMt=ASUn3EhW6u$mmK)XPHc=@mTn zxoEt%P2d<%#u_|j>C^IkK>1pey&vT`q{&_j_|o@HWl4zsmnf|)2Kx6bbQy^<)tk(0 z5X+X@_~yU&B!QDtTKq9=UgBK9P|>VZmRc(eC|B6`fojH%JO!ZoGrnR!n^Y__gW!O` z>Vn|fG{1puU@&<#HjuK)Rabh?<0*nxr0c+EgT^saz`jqt{t=Ne|HL3wPUL`{a%GKo7`l2VU6U zW6rWf9W^oUF&;8i!DN-OhwO-(OI`%LpnLq>>gik2Ag_Budv&fZwx0TRNfH_(Nbny1 zq??aCTq=x%gING>$sPuJ7UM>74Bv7BnFWNezT4R>!FSS3@Kq(}m1u~jdHeI+0APhw zM>{B9YFug{UMB%0af>>}60g(xTPg(KPEjN|@tu2_^{!>Q?@jAFdYD%HH=5-0I_1A& zHI;fyykPi2Rgydv{D2azz8#qqd_OrGv<^i&eRAT2oi!2skSg4-P?h{?pHeiE5}o>~2eRv&@6CjF zPFp)_jKCkLT8p%h!i3-pdeKjjn685PmOv;GBCo^-KdIY01uOx>p(>dH$hg{d+5V$d-$c>_3*a+k$l5K zaYiNj8`31QTx(4Mp)M}mh^&a*%J`^+tc#3!tbjM3xRff&rGJu86rN+pH%M0m2@k~2z}Kg7@Em0h~b7}P)3rE!V- zmHR#=89OrCXr>QiZZGSuc+#lA-e{hYtjr4IIh#F!f+MwpgwGBzvm#ucGJS_~r_{j- z5+hFSi=MiDj$0m{koRTVjRqxDp`y7L z=hWO94frcE0BUQmIh#RGBtO0)dPw6$=>k?!*)AV)4I=iqYgB=$!m6?jlL-?BaDygN zDCP~&v5XNnhW<_@8Cp}~t%)Plyai&&PNV-+QXv;iCgftzidheoS)-#SIlUh2ZT+^p z`jB`kMiuasP8`3m!^|~wYLnt;c!2&Qafo1D{(1Cx6JTGbyU-etu-`qQ3y>MqlzCmG zF#2e;As3rRYIqPhO?YU&KA1r82(=jzh>_>1;u+a<^DzX!xLR!P8;>o zV69=wOV0XkpSwsTcY?)nn{w8A9m_JGA+6QR7ft!1E9SoS8c?RToKXyNY!S`^nE9M( zY~;^-onuind)UIhe3wsh_FZg_F5_c1cNHiv_qM4r)G7u20(~aRnSdLQH92LK$fRVx zh{zFU-c%v+4@>G+_?fqoS4SnncwuFSa`%wfE0>lK5Cx;kJZ8e+TzECOqDtzCmoJYh z_m?XBQzb?vj4XUjm)#`hzsQN6{unjyueZ&*NyqH7RBU1Apz(@DUwIOwP>ewEk=x8} zBFlH_#zEfS5$xZ_i;Ije+DSJgX(;+9FR#G-NS`@kPj*nsWcVBdc*jiBLgxB zOW}lf3*%En7Z)@PWXeSbE@(Ma8Dap7hLdyG@_M!=;Q*(t@ZZyknrj%llWo?5U8#&C zA3;D;#p^1Z5{iR_7)1F#=+DG?3ZINSg?W(Df_ueq_LVj6^3RAn>ro#f@+k(8qLxl7 zcr_IBj6p8_v`tiAp@e%_j0QXC%)*MDx5UZele zAK)V9P)B1(1$JZ5#I%@#HFwSQ58%e2c{DZ|w=NjS5mbHFGRUkP<%kFhOj;hU1whjj zwmyXk8>0{{ihhp?PnZ3KZH&uPThAHdwN-#nvdq!7=_k18}vFKt_zc!}{4Q75qRW`}zV z#|5RS=N;_hn!{2=ztc~=J*ZZ6EV22ZZIQibV+)vk{EcNHkWYZVp8BzRqR+PhM-zTe zcP9ADTkRdwn)%Lpp{*aYn#%$D;y5LHP=4oR4kdEh9KMGP#7h%qGZ|-Km{1?{+o{A9 zaAizLp0O`JYZ)e@dr5EiMdy(er{e>D#;JJo2b~3s9^k@;iBKJ$vy=3Z{TRs+Y`6QY zv&QS^C^DhNTPtpu-~oJbD>-;qas4qlvm^TfF{p&6SDr%}yoja;yGS}b_K(nomc}ma zQ-@D%DpVw`=_jZ$Xb}?l(chr7dQ+U>C%Bslf&<%b*Sw#i)TX^Rc4NAc3 zGVo6aq|w!~VW?Jz*Fm<5*WRwnGAg8Nu)-ChM?2j6({uqzCYkWq^m2mI+%e3?mqdU4 z0m#3ooZ``gGs=BTNl}*yT?;gzK=IhYLrU~n0JA#;Qa^$ZE0JpC>8=nscU-gCebl84 z_$;@I5V(%FKM3@hiKMTGz}x*BdN%ZQ7w+D%L1!Yb8C*AbZgBhBy9e*wu*J?$w)2!z zLM^~G)KfzeJ>g_VuH&e@pQYI3t||xE+d-U?To%f$OT}`cmQ#R0RW9z8Rxg|0UY+L& zF|r6Qb8gC~YR%Y5c#>&~W^zNLu17k^)Js9}^R}hY!qUy%Co4lzMf#03WXg3_QLwUC zByYBimvel4pRM+LSVa%Gn;1^ESgW{BqHJiEG3oC^+z@FHan@g3f&FB>xZ!0BmIAsj z+b~8QvrWT<8hLW(nU`C2M5t4P(68L#1X3);Z(Uzr>9}3+k+B5HQdo!Kh zo*ZYT&`?Kp1B)jzKb*V`9vg znixGi$ePR49H8ueWMiH=GXibm3@+*knF&`uku0F?nk;1&zN+DCsg!}6*!eMvKW~ zlx+3d-un!FjlNDFp)b%GuCzALlP--U?5sGeA-Ny_8M61!;o#HXe-=utK0UfWr7QeJ zMho$gY)4>6Ogch$#qUD=Cd7x2U7>pdyUDxLdV50e3hX6szxD12y|?!6x87Z$57yp6 z>+K7DW8Lq~*1IS4;o5tv_3jIOd+oi`dJlyDLYxi1Ls&>} z+&lnYDy;cIAh(PlJ;FSo24msv(z|Rh;2RuKE6ymsz{*90H{)MW#gGA= zwNso)w!G^M|$@-U(K>!7CbahGwhJ*clw>k>vrO*YHp2^yfzzhI;Hq$%tw_n z1eiFVmb@7fJeD%j0!Ih59Q-z*t`7#acjFSP6~cni7hN_?^a@h?^}mv0K!QWSwIm&B1fZX~ruJa%d7DaTvo+~k)r z6}oxc5W_h|4uC$#K#DI3$Gp^F9c4AMsp=A0G#4h$s4{Q`2^b+ZV?j+h>GY%2!Afnd zwqzd{V8SV7NP9Z>wiO6Bdr}uLKfOv@1j$#+>oW`P%fGt6`BQMlTcv@l>SYT z)kfO~5h@wgEU}cN-vy>Z)+QyTVH*4qG(@Q|voIy+XL`_Pa!Ty2Qzw_#Z(F2X*KSbj zwXBStZV2>83ZxvpPJk;^3xG-)E0W7R$lwc3hwMt*Y&s%6U!Tp=8KK}RUvSMYxLOsQUj@skV3q}ke#Xr?owUAh&Q?G0 RB*Pst{odER>AT#6`kdmK5Rv-=|my}J4n(XDEtu{b>Pa*%BfPJ?RK>sk}EBD znVq2}1~%%}MS!+I(>Eyk0R7iD>Lc_4+H>wKNhR@bt3OJcquJS++1Y!~z4zS9`oGs! zfBmn&hl#2_b^N{lR4MyUbb3ndp$$~8rm{edYHF{h0$$fuR#$s$0nIL)6+?uW$}Jy?3K>y-M*d{lgp*i6PxA({b1^bMXs-Pu4-(RnNvB7 zw#KXuog079?e>bqcE&QZQitCZU@lcDTYl-^nuUm{=H!HBTd#^(FoiZL7d6llWkxqX$()~Fg0ioFL)y{h~g|4>^UfI-{ zt#OKIA6|4jafy9$>#{H+b1NX$YJW%RMot@tR z%mH1;lZJ9Nbs!#!6M(FmdfZTtF%1%`seS{~YEPQ#F}zSyxfZ=pz>6iW4={J9UB~wJ zPiREo6#9;f;ZYXw5EObnaJPsin8LMyXgS6q6l$X!`vx}OzCfFAr&4orVi!=-D>q5iYTzs6w%L&U+|^?arecc?*sru?y-i{ZW_}H+pYlif^*QP+d z2e`!E93Q8J2%ZOG$lu3bOKVXr!L*<}igJXbC=yN+MWdoO$#_Z7M-lj06ooXV@VjhC zS3-J<#k)xFg~qy&R?6d8@~~D5&Z@DpwsJPXm3(N*H=pAEK04;P)M;7R6DTzJGuX8d zC^G7$8Hd7zrjRL6Zm{WPWmd?#!K|NwxPyT&$=x%gS`rhk;aL^!nW4LiLxRJT%u29W z1vv@NbY4`9_}914afrSWl3|ZgZAMG#GR_!GJB`5i=>NEQ+lL)?#lhWvgfQwTqbh{? zz!*j3tW)uPv{5lp@mTkYf++0 z(}EJ8z;6jnK3rDiTeG`xV~`eZRBr4aSU6`WyD=yuQ=Wxu@{a54&Z%M>;B+8qa2{m7 z#l#_%$dT0`W$7^!qF>}Etmp~}f=;~)^KOrJn^DJ%9yf>^0! zwqv=e9(=y3@P9X-r;*5URiHrrd`JhH7a(^tqNu0FbtE1%upA`6@9t5*RVC{EH2E`$E(Ja`vG_><3Y`~PfMvf7jJNl+;+hu-!C zfCb(HbV2s3xck4xl7-cdaWKouoocPuE(90qtHEo*wp2$$jnVLRHa}n^wtI!WHX5PQ z$Qe-5bJLJHSNJv?*6zHWo;qo!mE6ErlEGFTq`7(a%zATWX=Uv~Ex`@>&@l;h=Bt_K zQs-|~9l?OPxqStTZ)Gt;u`e#7#K~%0>1(n`r+w+OqwkUn`P!f_=@``dq@u@yr0}QG zY@xu4nhy!te5satA?bBuQq*h*+M+bs(VyB0JjZmttBVhjW+pMJjzwSgL-jY-c5D)q z07wqTD6$zn0&hS$4b`Q1u&MXKZ8D*Ghd2l=I2qkb9!|?UTb8 z-f3t`>Buj7i)j)_vXr7kLPl^2W^b!k3~BB;}s~(UHEjp?+T#St*fiiEc@j z`ZVSucj60=v)F-^k3vpzG@*Wad>6k%_`>P!8KmwM<%2Xi%uGlL3Cj{ved^>TWP0*P zeCIjL#&ab1Iwqu&v7p}iu9qQeW{{HCl9ZHF z%Q;{BO`-Fx-yH;DQIybHk>CriXv1F6FQKXpcA^cY4RX(SS2CC$r" % (self.__module__, self.__class__.__name__, id(self), self.type, self.flags, self.seq, self.data_size, len(self.annotations)) + + def to_bytes(self): + """creates a byte stream containing the header followed by annotations (if any) followed by the data""" + return self.__header_bytes() + self.__annotations_bytes() + self.data + + def __header_bytes(self): + checksum = (self.type+constants.PROTOCOL_VERSION+len(self.data)+self.annotations_size+self.serializer_id+self.flags+self.seq+self.checksum_magic)&0xffff + return struct.pack(self.header_format, b"PYRO", constants.PROTOCOL_VERSION, self.type, self.flags, self.seq, self.data_size, self.serializer_id, self.annotations_size, 0, checksum) + + def __annotations_bytes(self): + if self.annotations: + a = [] + for k, v in self.annotations.items(): + a.append(struct.pack("!4sH", k, len(v))) + a.append(v) + if sys.platform=="cli": + return "".join(a) + return b"".join(a) + return b"" + + # Note: this 'chunked' way of sending is not used because it triggers Nagle's algorithm + # on some systems (linux). This causes massive delays, unless you change the socket option + # TCP_NODELAY to disable the algorithm. What also works, is sending all the message bytes + # in one go: connection.send(message.to_bytes()) + # def send(self, connection): + # """send the message as bytes over the connection""" + # connection.send(self.__header_bytes()) + # if self.annotations: + # connection.send(self.__annotations_bytes()) + # connection.send(self.data) + + @classmethod + def from_header(cls, headerData): + """Parses a message header. Does not yet process the annotations chunks and message data.""" + if not headerData or len(headerData)!=cls.header_size: + raise errors.ProtocolError("header data size mismatch") + tag, ver, msg_type, flags, seq, data_size, serializer_id, annotations_size, _, checksum = struct.unpack(cls.header_format, headerData) + if tag!=b"PYRO" or ver!=constants.PROTOCOL_VERSION: + raise errors.ProtocolError("invalid data or unsupported protocol version") + if checksum!=(msg_type+ver+data_size+annotations_size+flags+serializer_id+seq+cls.checksum_magic)&0xffff: + raise errors.ProtocolError("header checksum mismatch") + msg = Message(msg_type, b"", serializer_id, flags, seq) + msg.data_size = data_size + msg.annotations_size = annotations_size + return msg + + @classmethod + def recv(cls, connection, requiredMsgTypes=None): + """ + Receives a pyro message from a given connection. + Accepts the given message types (None=any, or pass a sequence). + Also reads annotation chunks and the actual payload data. + Validates a HMAC chunk if present. + """ + msg = cls.from_header(connection.recv(cls.header_size)) + if 0 < Pyro4.config.MAX_MESSAGE_SIZE < (msg.data_size+msg.annotations_size): + errorMsg = "max message size exceeded (%d where max=%d)" % (msg.data_size+msg.annotations_size, Pyro4.config.MAX_MESSAGE_SIZE) + log.error("connection "+str(connection)+": "+errorMsg) + connection.close() # close the socket because at this point we can't return the correct sequence number for returning an error message + raise errors.ProtocolError(errorMsg) + if requiredMsgTypes and msg.type not in requiredMsgTypes: + err = "invalid msg type %d received" % msg.type + log.error(err) + raise errors.ProtocolError(err) + if msg.annotations_size: + # read annotation chunks + annotations_data = connection.recv(msg.annotations_size) + msg.annotations = {} + i = 0 + while i < msg.annotations_size: + anno, length = struct.unpack("!4sH", annotations_data[i:i+6]) + msg.annotations[anno] = annotations_data[i+6:i+6+length] + i += 6+length + # read data + msg.data = connection.recv(msg.data_size) + if b"HMAC" in msg.annotations and Pyro4.config.HMAC_KEY: + if msg.annotations[b"HMAC"] != msg.hmac(): + raise errors.SecurityError("message hmac mismatch") + elif (b"HMAC" in msg.annotations) != bool(Pyro4.config.HMAC_KEY): + # Message contains hmac and local HMAC_KEY not set, or vice versa. This is not allowed. + err = "hmac key config not symmetric" + log.warning(err) + raise errors.SecurityError(err) + return msg + + def hmac(self): + """returns the hmac of the data and the annotation chunk values (except HMAC chunk itself)""" + mac = hmac.new(Pyro4.config.HMAC_KEY, self.data, digestmod=hashlib.sha1) + for k, v in self.annotations.items(): + if k != b"HMAC": + mac.update(v) + return mac.digest() diff --git a/contrib/site-packages/Pyro4/message.pyo b/contrib/site-packages/Pyro4/message.pyo new file mode 100644 index 0000000000000000000000000000000000000000..37119794c6d8094a2af68d3b42efa355f1119d8a GIT binary patch literal 8857 zcmd5?TXP)8b?(^(78im9Nr{AH%2v}d&1IPa5}A%FjHD0(ArTgCnMFuz%l6b@r?ESW z-I?X|EWu?FRjSY_Kcp&8{sZ>k$X`g6pZp8_l;=F8DpjuJ`%cg7E*BIWP4Y)tguS1=Uo&$lP~j?ssJF4TZJLd+G@Lxukxj)GMXFZhn`WYt-Mu_N_&+c+fF=Fv?PW5@)Do zX`Z&yp6(lK!?synT-;~phkAdMchW@rrl00U?;UncE4Qm!Z={$Vw>!B$9Ow3(`2U++c&7Q@HSxroSXLT|DANSJ<(DqLn6S zPI8;q@MN+q&1^o0>W<^hUPPVT-CHfVjY%{nftZSpXIRX#AcYtoHMWF#JbeBQr0`o* z3@ia+LRk^Lp@j0GbEp~`hGOSc8@fRwbTgyOtm@X3nGxlo7pj87p`i=vS5W676$Xhz z&Cx}vK+~cFYI<2(E-3>|UuK;aa9JKf;R-3R1-ic~Bd;pIQGXj;+6C->QR7E+a+1Z+ z#J6aSBc4#3g;=bHDDyB*tWJ+~sE^|3Cek*3VRXlYk;(4rFwyBCkJBXVX=`#Bmn12) zf;Qdi43lGfSC$TwNar1B8HFB%qh1EY%CG)lq6}ayy49fFL#e+VJ~A ztsa>>)?{(mi(k04NWX)k7sGcT=~;8qF&T`LJ6%7|rw+08H0h1N zUS>^nSDWW8GsuO7+mCkEH*UKf-FL*vbY@_G(RhQ;zz^1hlnzZR99pB}oa>_&>Ci%k zg=lpYw~fu=CS--aqGZFeBwygZ|{HY@7=y5Q_%DgK4UFDGns|R zr~AsK^3lM2Tl-NjY}@G$Yn~2G(lR<3_76=~c5!(TWX~k+yi+!Fd1*eYhhhWuvejhZ z{S43HSnyUmlDeM5WO`yfy~gzcFv-(ZDZl|XUNvsmOV3P7i)D)>(~za zPMGUQIryOkmInO70HxAQ_fshU^@aN32rB9I(v!Q+s7#@u*VHpD7P0$cZ)N*)i0eFk zk>BAcH}))hVSZPb3|sjSVwv_9pe2SEJWZKTsXM2-vCZWvqzMU7l90~;(0InZGuH^$ zOu3>kTdbPV8f;!#Y~iMm{4+@23V${Rf8y!lR|UMJ!(OBZChLR)i$(Kc=FE~}AM+rV zbCjih2qi^NhAufFS|Kc;e$t6s9epCC(9e0p9^4|qa4nFoW!6b-%}r+p4n<@JRvrLp zV2YT*vj8>?0gwTkwvK^Q&TwG!eeMUZN$`{6O=6k34lA;8EE zGp83>?gUl}OC>%M=ma7ml0s*#6bfzSnm{qZhGGdKBxNaE$?%l+Wose}A@1ZRTtI4B zK0#myJrj~~1S5!#Vv`j+UQcuZk*^u~_zX#fL|$r5~qX@XWn4WnIddKu>5u3 zbSiYi>tmo!Vcx|A0=9z^fUq^aBZLs}L7nq-Cnn}R;ROWqDo;uCRJzE9#1X6nJneD@ z3==#H;wZm>mdQqKe);UPOtVX9ym`+)!cQ#4f80YL!AOKb1iv@F0bvi($4gJWR4PDa zMk;uml}bgu#2>n<3f!U6t@4y-3kFoBQd2K$YVe4G3ft#YKCik9s=KJNflRr;7>;u# z()=XB{+Cp{Z0cTQ_rqUYSEuN&$U1XeWaXC?_4HM_9HLC{;3zMvnd-4p>AwqA;k0{6 zt)MBXsEU{UNkJlAL?Vvm@qqF4ql&tY75>RnUCF8vDDSV%SqQA&yj*@6l!bKSC zQQQWfN|BG?tIe;4DcY?wlVM*5;;08a`ZNU5{?7W(f}PDqWBtp`pt055^d%mZP_u}Q zHyMf6Nm(e;Bqc2E+x7v?ku1^X#HJmClN|`F3{#)gQ79>F2emcCUu(S-u(`G!A7)`T zTKggk`{pFgj_sO@zE_XRmbJ$&POxhqt$w`Lix1ZZ5})%4Xl_0ngso%5@0^JMd9Cox zRg5CzK@cZ#9t4k3x9^}(-a_$v72nI=l6T$viFem)kyU=kw^;c@RB%P26*|N7R2Q}; z7VHeVJ8dkmX|Y!<%gO?+@MvCLZ%CZKYwv^HC%5eCEud2^qoww0JZtaAkiBU z(0Tp{q|05P2uTb=-_cZ{V{x0q-60v>IWvIssF->ctK<|~u;XrLLEw(7K-?fFtCOxN z$aVeV+5bNzpQnx`;A$cHig*E~&XYhS|7%p7)7|>H?1#Gsso9 z&fB%Kb$YO%+rGbdu(z?d9X#Ij8(Vw3&g|znkO+7Vt>k`Q20lSQY{xCZOxVw`l?(b2 zXXIQhwC%19g3aRL;z;6zC7n0rRg}hAqr@?%ETyNsht9kBKw*fE{{To~^BOpLRn{u+lAYRE???GW0fx{k6-qZKh69|(>c|{#p zRQ7^8?LyZW@2ST5heFT7FM|;|#9F;L2bZdT_87zg(05rA37{kYhZy7EWcR3n{KBY;XZy zKeot&v^xF=T>UPJIFam%i~Di28zy!*7^E4qE$7o9f@%_=6og9<(PO$KS(n3GOq=>H zpoqbOL~s5zG>Q)hdJPkTmQQ2$Y1{q}SlnX4Q_RJcq$6YtKeIsUlJZ0BPMY(uE#FAm zh%X@m3>dylMXd9rDBekm;!A;lhgGhH32(}$$PVxA#WDTQc(gpfRrNjZs`q2>2Ff*j z-jTSlMQ$kp?nnITeRx=LPyyClKvWHv$M~G4j;tw5*!yvL$M_vx z=1kXkB5xOPHOLl$BLERPaE?p2Sb^=YU@?JJxfyx54dD`m_OTDcA+tXU^-FZl<2gPe ztIPE+f%t4e9PUF>1%W9abKGVpvI=%7%yWx1xu#WBfGjQ^=hXAIIGzl^<}&LQcPDHa zfAC2_JvM%mK!OuPV8ivWx#%Sf+~g|D^y*JsD;qz?5>GFCPjJtI3@C66lD7w_1b9UN z43KHZHI@CrD`fZ<>#iLG-ev7=^$K_nNV!qrcJV4ff^uNe(;JoKi!+1}(TxdV({2{B z-G$s%2s@?Rt`!djxbZCmZOr}`Er%C&{`jV4;%5Rr1HM-%5y)kbbQ^sMv(*W*A4r3` zn?5J5x0u3pXyitfQ|WMsPIr+S{uJ>k@_QLFZ2&GHP>~2NG6q7YXH#ZnkfWJ^+BBvy zqWzY9-Gr0JMQ)f2Oy?$!fL@Z#oD7vP>@U6eT1ssq0xi-1eT{Q#f}Ow(xhHAdg^(zs zNTYFNAa`z`p{gbDm2#4dO$nR!yXcXYV>6<_aandrrBT0c@+@xEc`fumLm`m?R(a_GFIA*9N_bnHk{0$#@LitR=_*YO! zSlKYGA>MP0GRP9Q){Bqsqcv!8es679Ilx%EHgK%$ti zh-Tpu4>EjW6E@%>CV(Bwy3)lggv)imA>8)=9vu7zA4`d;`5U;TYwvw;5m0?iLLXe< zE4;+l(DDKLu2rr9%-==t%}Na;u1-Q-rX>W-yQm;40;;$$Xnchyj7yOEs^T&J{ae9& z!Q)rVHPxr$-!k^e?g>HxtBXkYo_cyuJt1ldOgH4RXrEwBmsNINE!J4%{3L#0gdyH| z?zMq-DH~m83#XmXB?w&#M&8UIDY+$yG_i z08u*71HnY>SJ<04kktN9lVB3)aTkKBwI9{I@4;V|y;^1I(n57%2EW&rm#g0L52s!! z6R~0%!`fGU5DtpcDDc=D1`r~q$E z3Kifr#x{nV%zwaco(4OOFN2M}-QCTN0|CXU=e@6TV)0bV7wcQw!WY}Nb|3G3wK;zD zHyi)BeIOcT@BY^Amooi}?e#Ak!RF64Hun#Z6%Z(NZ5w+#`=(3, 0): + basestring=str + +log=logging.getLogger("Pyro4.naming") + + +class NameServer(object): + """Pyro name server. Provides a simple flat name space to map logical object names to Pyro URIs.""" + + def __init__(self): + self.namespace={} + self.lock=RLock() + + def lookup(self, name): + """Lookup the given name, returns an URI if found""" + try: + return core.URI(self.namespace[name]) + except KeyError: + raise NamingError("unknown name: "+name) + + def register(self, name, uri, safe=False): + """Register a name with an URI. If safe is true, name cannot be registered twice. + The uri can be a string or an URI object.""" + if isinstance(uri, core.URI): + uri=uri.asString() + elif not isinstance(uri, basestring): + raise TypeError("only URIs or strings can be registered") + else: + core.URI(uri) # check if uri is valid + if not isinstance(name, basestring): + raise TypeError("name must be a str") + if safe and name in self.namespace: + raise NamingError("name already registered: "+name) + with self.lock: + self.namespace[name]=uri + + def remove(self, name=None, prefix=None, regex=None): + """Remove a registration. returns the number of items removed.""" + if name and name in self.namespace and name!=constants.NAMESERVER_NAME: + with self.lock: + del self.namespace[name] + return 1 + if prefix: + with self.lock: + items=list(self.list(prefix=prefix).keys()) + if constants.NAMESERVER_NAME in items: + items.remove(constants.NAMESERVER_NAME) + for item in items: + del self.namespace[item] + return len(items) + if regex: + with self.lock: + items=list(self.list(regex=regex).keys()) + if constants.NAMESERVER_NAME in items: + items.remove(constants.NAMESERVER_NAME) + for item in items: + del self.namespace[item] + return len(items) + return 0 + + def list(self, prefix=None, regex=None): + """Retrieve the registered items as a dictionary name-to-URI. The URIs + in the resulting dict are strings, not URI objects. + You can filter by prefix or by regex.""" + with self.lock: + if prefix: + result={} + for name in self.namespace: + if name.startswith(prefix): + result[name]=self.namespace[name] + return result + elif regex: + result={} + try: + regex=re.compile(regex+"$") # add end of string marker + except re.error: + x=sys.exc_info()[1] + raise NamingError("invalid regex: "+str(x)) + else: + for name in self.namespace: + if regex.match(name): + result[name]=self.namespace[name] + return result + else: + # just return (a copy of) everything + return self.namespace.copy() + + def ping(self): + """A simple test method to check if the name server is running correctly.""" + pass + + +class NameServerDaemon(core.Daemon): + """Daemon that contains the Name Server.""" + def __init__(self, host=None, port=None, unixsocket=None, nathost=None, natport=None): + if Pyro4.config.DOTTEDNAMES: + raise PyroError("Name server won't start with DOTTEDNAMES enabled because of security reasons") + if host is None: + host=Pyro4.config.HOST + if port is None: + port=Pyro4.config.NS_PORT + if nathost is None: + nathost=Pyro4.config.NATHOST + if natport is None: + natport=Pyro4.config.NATPORT or None + super(NameServerDaemon, self).__init__(host, port, unixsocket, nathost=nathost, natport=natport) + self.nameserver=NameServer() + self.register(self.nameserver, constants.NAMESERVER_NAME) + self.nameserver.register(constants.NAMESERVER_NAME, self.uriFor(self.nameserver)) + log.info("nameserver daemon created") + + def close(self): + super(NameServerDaemon, self).close() + self.nameserver=None + + def __enter__(self): + if not self.nameserver: + raise PyroError("cannot reuse this object") + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.nameserver=None + return super(NameServerDaemon, self).__exit__(exc_type, exc_value, traceback) + + +class BroadcastServer(object): + REQUEST_NSURI = "GET_NSURI" if sys.platform=="cli" else b"GET_NSURI" + + def __init__(self, nsUri, bchost=None, bcport=None): + self.nsUri=nsUri + if bcport is None: + bcport=Pyro4.config.NS_BCPORT + if bchost is None: + bchost=Pyro4.config.NS_BCHOST + if ":" in nsUri.host: # ipv6 + bchost = bchost or "::" + self.sock=Pyro4.socketutil.createBroadcastSocket((bchost, bcport, 0, 0), reuseaddr=Pyro4.config.SOCK_REUSE, timeout=2.0) + else: + self.sock=Pyro4.socketutil.createBroadcastSocket((bchost, bcport), reuseaddr=Pyro4.config.SOCK_REUSE, timeout=2.0) + self._sockaddr=self.sock.getsockname() + bchost=bchost or self._sockaddr[0] + bcport=bcport or self._sockaddr[1] + if ":" in bchost: # ipv6 + self.locationStr="[%s]:%d" % (bchost, bcport) + else: + self.locationStr="%s:%d" % (bchost, bcport) + log.info("ns broadcast server created on %s", self.locationStr) + self.running=True + + def close(self): + log.debug("ns broadcast server closing") + self.running=False + try: + self.sock.shutdown(socket.SHUT_RDWR) + except (OSError, socket.error): + pass + self.sock.close() + + def getPort(self): + return self.sock.getsockname()[1] + + def fileno(self): + return self.sock.fileno() + + def runInThread(self): + """Run the broadcast server loop in its own thread. This is mainly for Jython, + which has problems with multiplexing it using select() with the Name server itself.""" + thread=Thread(target=self.__requestLoop) + thread.setDaemon(True) + thread.start() + log.debug("broadcast server loop running in own thread") + + def __requestLoop(self): + while self.running: + self.processRequest() + log.debug("broadcast server loop terminating") + + def processRequest(self): + try: + data, addr=self.sock.recvfrom(100) + if data==self.REQUEST_NSURI: + responsedata=core.URI(self.nsUri) + if responsedata.host=="0.0.0.0": + # replace INADDR_ANY address by the interface IP adress that connects to the requesting client + try: + interface_ip=socketutil.getInterfaceAddress(addr[0]) + responsedata.host=interface_ip + except socket.error: + pass + log.debug("responding to broadcast request from %s: interface %s", addr[0], responsedata.host) + responsedata = str(responsedata).encode("iso-8859-1") + self.sock.sendto(responsedata, 0, addr) + except socket.error: + pass + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + +def startNSloop(host=None, port=None, enableBroadcast=True, bchost=None, bcport=None, unixsocket=None, nathost=None, natport=None): + """utility function that starts a new Name server and enters its requestloop.""" + daemon=NameServerDaemon(host, port, unixsocket, nathost=nathost, natport=natport) + nsUri=daemon.uriFor(daemon.nameserver) + internalUri=daemon.uriFor(daemon.nameserver, nat=False) + bcserver=None + if unixsocket: + hostip="Unix domain socket" + else: + hostip=daemon.sock.getsockname()[0] + if hostip.startswith("127."): + print("Not starting broadcast server for localhost.") + log.info("Not starting NS broadcast server because NS is bound to localhost") + enableBroadcast=False + if enableBroadcast: + # Make sure to pass the internal uri to the broadcast responder. + # It is almost always useless to let it return the external uri, + # because external systems won't be able to talk to this thing anyway. + bcserver=BroadcastServer(internalUri, bchost, bcport) + print("Broadcast server running on %s" % bcserver.locationStr) + bcserver.runInThread() + print("NS running on %s (%s)" % (daemon.locationStr, hostip)) + if daemon.natLocationStr: + print("internal URI = %s" % internalUri) + print("external URI = %s" % nsUri) + else: + print("URI = %s" % nsUri) + try: + daemon.requestLoop() + finally: + daemon.close() + if bcserver is not None: + bcserver.close() + print("NS shut down.") + + +def startNS(host=None, port=None, enableBroadcast=True, bchost=None, bcport=None, unixsocket=None, nathost=None, natport=None): + """utility fuction to quickly get a Name server daemon to be used in your own event loops. + Returns (nameserverUri, nameserverDaemon, broadcastServer).""" + daemon=NameServerDaemon(host, port, unixsocket, nathost=nathost, natport=natport) + bcserver=None + nsUri=daemon.uriFor(daemon.nameserver) + if not unixsocket: + hostip=daemon.sock.getsockname()[0] + if hostip.startswith("127."): + # not starting broadcast server for localhost. + enableBroadcast=False + if enableBroadcast: + internalUri=daemon.uriFor(daemon.nameserver, nat=False) + bcserver=BroadcastServer(internalUri, bchost, bcport) + return nsUri, daemon, bcserver + + +def locateNS(host=None, port=None): + """Get a proxy for a name server somewhere in the network.""" + if host is None: + # first try localhost if we have a good chance of finding it there + if Pyro4.config.NS_HOST in ("localhost", "::1") or Pyro4.config.NS_HOST.startswith("127."): + host = Pyro4.config.NS_HOST + if ":" in host: # ipv6 + host="[%s]" % host + uristring="PYRO:%s@%s:%d" % (constants.NAMESERVER_NAME, host, port or Pyro4.config.NS_PORT) + log.debug("locating the NS: %s", uristring) + proxy=core.Proxy(uristring) + try: + proxy.ping() + log.debug("located NS") + return proxy + except PyroError: + pass + # broadcast lookup + if not port: + port=Pyro4.config.NS_BCPORT + log.debug("broadcast locate") + sock=Pyro4.socketutil.createBroadcastSocket(reuseaddr=Pyro4.config.SOCK_REUSE, timeout=0.7) + for _ in range(3): + try: + for bcaddr in Pyro4.config.parseAddressesString(Pyro4.config.BROADCAST_ADDRS): + try: + sock.sendto(BroadcastServer.REQUEST_NSURI, 0, (bcaddr, port)) + except socket.error: + x=sys.exc_info()[1] + err=getattr(x, "errno", x.args[0]) + if err not in Pyro4.socketutil.ERRNO_EADDRNOTAVAIL: # yeah, windows likes to throw these... + if err not in Pyro4.socketutil.ERRNO_EADDRINUSE: # and jython likes to throw thses... + raise + data, _=sock.recvfrom(100) + try: + sock.shutdown(socket.SHUT_RDWR) + except (OSError, socket.error): + pass + sock.close() + if sys.version_info>=(3, 0): + data=data.decode("iso-8859-1") + log.debug("located NS: %s", data) + return core.Proxy(data) + except socket.timeout: + continue + try: + sock.shutdown(socket.SHUT_RDWR) + except (OSError, socket.error): + pass + sock.close() + log.debug("broadcast locate failed, try direct connection on NS_HOST") + # broadcast failed, try PYRO directly on specific host + host=Pyro4.config.NS_HOST + port=Pyro4.config.NS_PORT + # pyro direct lookup + if not port: + port=Pyro4.config.NS_PORT + if ":" in host: + host = "[%s]" % host + if core.URI.isUnixsockLocation(host): + uristring="PYRO:%s@%s" % (constants.NAMESERVER_NAME, host) + else: + uristring="PYRO:%s@%s:%d" % (constants.NAMESERVER_NAME, host, port) + uri=core.URI(uristring) + log.debug("locating the NS: %s", uri) + proxy=core.Proxy(uri) + try: + proxy.ping() + log.debug("located NS") + return proxy + except PyroError as x: + e = NamingError("Failed to locate the nameserver") + e.__cause__=x + raise e + + +def resolve(uri): + """Resolve a 'magic' uri (PYRONAME) into the direct PYRO uri.""" + if isinstance(uri, basestring): + uri=core.URI(uri) + elif not isinstance(uri, core.URI): + raise TypeError("can only resolve Pyro URIs") + if uri.protocol=="PYRO": + return uri + log.debug("resolving %s", uri) + if uri.protocol=="PYRONAME": + nameserver=locateNS(uri.host, uri.port) + uri=nameserver.lookup(uri.object) + nameserver._pyroRelease() + return uri + else: + raise PyroError("invalid uri protocol") + + +def main(args): + from optparse import OptionParser + parser=OptionParser() + parser.add_option("-n", "--host", dest="host", help="hostname to bind server on") + parser.add_option("-p", "--port", dest="port", type="int", help="port to bind server on (0=random)") + parser.add_option("-u", "--unixsocket", help="Unix domain socket name to bind server on") + parser.add_option("", "--bchost", dest="bchost", help="hostname to bind broadcast server on (default is \"\")") + parser.add_option("", "--bcport", dest="bcport", type="int", + help="port to bind broadcast server on (0=random)") + parser.add_option("", "--nathost", dest="nathost", help="external hostname in case of NAT") + parser.add_option("", "--natport", dest="natport", type="int", help="external port in case of NAT") + parser.add_option("-x", "--nobc", dest="enablebc", action="store_false", default=True, + help="don't start a broadcast server") + options, args = parser.parse_args(args) + startNSloop(options.host, options.port, enableBroadcast=options.enablebc, + bchost=options.bchost, bcport=options.bcport, unixsocket=options.unixsocket, + nathost=options.nathost, natport=options.natport) + +if __name__=="__main__": + main(sys.argv[1:]) diff --git a/contrib/site-packages/Pyro4/naming.pyo b/contrib/site-packages/Pyro4/naming.pyo new file mode 100644 index 0000000000000000000000000000000000000000..ed929bb8bfba86c5e0f617866ba599dd4829712b GIT binary patch literal 16892 zcmd5@TW}m#T0T9a+l+OyY|ECN*lov3#z{0zve{g`Nj9>rBu<>M(<4WYlI-kgx<_hh zrhC+<$CgqSc$4fx6}4mw9=PU#q9_WArJ#TZDm<{yTacx8Yae*wfh`K2sDcL`xPIS% zx~E6BLoF{BlKSZMxnKUv_y6Z~{$Iz6U;F#tavCc6Dd7Lhx0TYrX({C@wT#eGZceo< z)y}EqoU-_ySFOBS&PyKR18R97jSs5j!8AUkmWL!hpjyMKJ))LJRJ)*-3#wgI%SF@H zplahKrJ6sfl}UpT0f-RQT36fmZz0Fro6+diSduCkCpmJsdtu-D0f161+_jS z8I!$?qsm3YrubAZ^%&=lKkvaLVqO^@P#|%X8#xvIs^#GovpcQa8Fd2FurMw3RxKUFPU$X!Vx`{p?5Y>t@FKe&xb~XY z>LA+e294Mc16?i_uWm=7J!fCtj@QD#cD!~Nd-j!;b*~ZYvTa`oJKK@Jx)$3j+xDfX z?FF{$*{_Ges$KH=`f^nNAdGJKfmaT^_)NLz|2KY0pb-_1Be?0uYc(C$V-JPnN-56_ z(ozHc$MqoA@qoNVUR*>(hm8#{?#6zrlw;9B1kPn7#zTlLu0>wmH92gAqhE-kFpBr# z4b%05Rr5ZA_g2_Izm;k{jF-sMVe5t`adaHTm8#Fd_;{2aMZm_ME(g+xjD^c*RD7XE$5**fwQ4^@eB1q1~=`>{hty zH|i}rl%13Unt9UXHRqBpH@HQ7xXc%BqstjYuo}0Ns$t}=SH&vH{*XD|CJQ9ar|`$~j%Zidl@K5Iy@+)Pu> zzU@VtRC@NQ^0Q}K{>s^o5GmiZAA57?eWSkWu_iX|EJlRgFQX6_uvYT}Kd#j{Db4XH zt6()aS3aaFne)qtY#&jvQtK9HzMfOJEHwjmsxAd5)j*tA8$}g;MOl`GNn)3geHhsv zS}L&Ab?kOkXj+%i(&aGR=yvRQ&9hhi8{ieL(&IKryBh`Aw7|yF+kVq-hTXu`g20l=4i+bCqdKB7LEQWlyuWs(2F61erP z>jVXhUWzV(a zn|{M97wzO{5hT-%e3oUwx~<~~)NF?o+(I>mlgs*H3?d9#+f3BVHzjpiHS0r9p{R_Y z-PO{+G%I{A$@Lbc>2}|+NP(2ABK~~si--n~lOUZ*W^gC#x+csIxjAPq(NH{A0WBtFViyX zAn5^lfv^FUf|!-HK1kjGx$g|5)IX3A^DW?*p_G>l2G}~Hkb3d`NwqU5Euq6f_2TBQ zsU?s;vd~b~bf2)uOENBua98gb26^&}d!Mxa54|6@?$x_H2;SnF>aQRm`nW;jH#-v5 z2~x^KGsxiF(p43kdUJhwmW{y6C(En zbdy>%ScFp4+iC!EHA4~k!YP=5lx^4J#+ooh1J*mb=>l{bl zm+^?ERRcvZ$q<-l0*HnTG-Tzi!+4*xj^<42aK6FurjM` zI5bH_%V}TFoITItN&TGV_&sOMA{yzH;R% zVc>}fbGRAK69}@2O6-2x6AmXvTp)lQ6p$+n6oyA9r;bb-tf(Fv$ zhh%;JHY&~|;`LjLfVBO8FdpF@fyy_=qp=)O9+o2vMnr~b7_vD%i@9?o-8kdYyg;H=|E6N$0&C*U=1-Zw(Q1;ik?Y}4XPajq{Rqi)>~=T zkbrg6!l;3Jkkv}FhN%vSEiez0_3bokM0I|Si4fl?0ND*1fqDUYhrGoAh`+^#X0y;6 zknH)K6~j-1=0H70I**HthAA+i)+bq60L++LA5W5~vGym0pc+^ro>J=vxSV*mFam?= zE1Ik>%a*X+%`kW*hVX?1v<2K>yt25sa8ba5?FIFf79{vrT_hJR5yuaJZ$;msxxO z!E0Bli_%S{TDyA1N#gU1S^E5vscx%lLDgXOZqahaltQ`^v(cE(lI(3KV6HLY(h@ z4p}9nOk1a{o-pTpFWg2Q8ximgG-*PrMCC$6BuSzF)IY*ic3x({RhA8Hv_i->ZkR(c zARdxEl-SToO+KPiXd-h{ka}eSC}bfcxV`W$z*zxNWTX^u zYAU5ZqO%nu6>c%v_TCov5x@wt(d`ypQJD$c157cjWf>aL6L4AAyu}e5Yb$VX?Gd-U zj#2+So}S~ZV(h0Uk>)(b;7bf-!XijWQ_dv@teTBlV)vEN$#yL~nqrM;?Z|#;>p^G= zsNyKByN$Yzjk0baNq)%a=WnCV2Z(s0rakdUfduGlhie~NP?RNg^gj86riPOq3K~9M zQP7E|hEW2iE>ilOx^VU~1r}KD3NjMcOr3-vW=E6RPpM|^!y=TbSt^bq)+e~u>~+lG zfk|1s{am@KVAqP6O6LU6o;wF47rR01B(kmUx{>JF*l&AbH(q5neqJ_~75C-cS^fUG zSyxjS%xcC?A_io(lG0rY?GvSM`G%{W$E$M=fs6pU{rm;d;vA2> zQNKG2EJf;#4u z{qsuE1t)H{m_!M0j~9e{zGD=Ce~hPpHXJC$k<3Psc{5P(G24Y3^Hsu zs2Yk0%7@7i{#6kv`ghTo(={%#yQC-%ojNxBXMPOOi6gXF4u|Pu0}91o8)h-iNt*D6 zy)Gw*kM}%hH`n~en!N^BT_*}*(ZQ1}wqBb~I2@~P(d**Jc9*;dsojG2rF6!Wp`WJT zU_|#g>MRS#!D^&TeGJ3C_i$5>4u+i#xl$VDJgEX=8eO$ahReBctrmHkUAT>KzSt3V z)?RFU01nTIMI?yDYUFesX?^E|BpS0edn1BH)nMY60&{%-rx=epCd4_*7*AHn0epHK z&IQ~u(iQr5Mz}&)-P+9K7(*aPB8^3TvtmWm?Y);Bs2_*JI8T5ir2-SFDfbhw{|!%T z$1HH)`f)saXr{#!2D>)?FOZNJEu(PW%c7BF_z1)O6C&Gxok7YK8iu)KVODrSOYqbHKe9=&tSHJS1jgbOLI6e4&Qfr&zl)!wI#jlt zR9*2ARhr!ML|J|XwP_kAa3^=dz$O2}%bJlz!b)~C3fn+m=O95q9ZlYynAKu6_&S_> z?zt~NKlfyb-n~q@!UubU)nsp+g}1IPR1H}tUaY7_qN;{bcjwHP`*s|!~ z3yhIv9a6oB5ie+ju4i<(7r1d~3{2`q!C!727ivhyg4Jj?nAvJia+uJYRn7104cj}| zLG?X6iO+>s0w#dACUSY;tT8yyhpnU5A?rcJgqX>3e1w=eJLtkx1>$Yb^|%CkMSf^s+7c>&+Pm&$P-}M z!962r(zdgSlshf|Cev0Kv>A{IW%vJoxCAN~&bRUCg9wJKW4Vb#g+d{Z-;u&#VXBaC zj39&Zmj8wNc(M=7kG-VSgrSR2;K?lh|5rIW$(HQQK zkkR+Wa}I+R0uvAlS@14k5D>ll)by*bF(O`bxfY0WXbGbjr9r}MypE~p-|5P@Rb&e8 zLf$Gc5}-qfX^DRTPdiGE3r1VQO!zh-Z|DH;8wC|TXR$Mr3ZB1?Ym;bh9K1{MsYznI z&qp75dh-xi4M!IOS$J9+csF-JD&AETR~>L1z+ED;iTBExCs{lt(CKo`Y}#kH6x<0hvriAKLvqe{%e}~p<<3sdTZgpZKa&AGkBFj?|?wGo;e&b&V+r)PFDf`5s^kK z4YO%dnESO?N=~LPG*zjRKK~Yj)wIi1?l^Sa5jij}z_T!64T@DeG%y8U!x-`k^hM-o z=Z@!!NCTw1OAh@$8v5Vm5adN7d*GD-a|*hG)PcOwpGu0jCn#p%=jY0c7BGrw;3!l z_yz+W#FX%u;J!M>is<#f@aSpwpFa*#ox%fB1yRCpj^76DDx%2(9#M=x&Wl#Wg>;_y z{$maOgNovIJSACMhS)a3CiWhT9nWj|w&iqiCzc<7H?~jN{__D;7?-x6fN! z-%U#&NFxVLiC>hq{Wh2KSP4X45OFwF1A)M z9#V+9C};7h$!;o7DO+kMlng5C5jq^j8KL9lry=y9+#~eA$y=ZkD}Iz8ldv)r0N7`r zsy)Uqy~TbQ!uu$ATvH9dDiRuwkuBpq13u%mk_0B{5Eu5>JX}zNl$JY7Ud#*pWpPYp z$OnQ#ojdoW93c{cId8Fj;xV3+2tv8~j&tSQtbW-Xme5jWb-4v6PPXc~)FhX5D27T^ z6H{i`&15(Z5}hi7)u#OX!&kF!JIu{cdP#6;EeMUvfWj}c^Tq3&^&#Cpj}mwGV>jz? zzPgX&cK0?e>hVq%@81OY4>ZK%9GuPYWJdf5+4S@YH1>+rP2nNFvPmy|nT4$wj~y6C z>16FBC7Vcz81~NNA*$S@Q-j^a)O?U7Uu1BBL7f3*xp)fk5y{rJT&gm6Nv<)MT{=q) zuBYXqdH`j85U-tjq`ky*;iZ?qMTo=PVCQk0uW2yQP`I2xS#xcXdahQJyFECsXM24pmYKK; zlU#NuK?#o-{}7kGCLOLc=#_EKB0)?PP@U7oi^J;@5HX;`J<|9i=z9VK(BDEtIPha=0F(Owykqj6X*NWCWRswSw%!=OM=5Yvd}S z0#Zx?K^`ch&(kkv{GG{{AL7@;Zz*VtMWn?w6r{Ed{UNQ3JW@|;C}PY^CrU_v7l zshgZuKw=_Az7#~~P|!R;hI*4WQ#U4hgUwOc>$IsTX54VJASXjPUGugCfXU#R-v;%Q=q<4MM zEDt*3uo1Qr7Z2ImA@b{UT^`0OG>_1=l-Vy<_mtIed)tLli(O!)uh~+>m=>)ih?8F3KD@_%%V&Jo65No zv*jRRR}*&Wi5H!asqPvR6{tBJo0~H?D8!x=m%?2$ z$>2$EPIk%lnsuCF(jRr|)EPNvV^cD{ud{KUGVGeoJ)6nqMgEzaOK!A^pGMnA>banL z!vY>`fc1GPwlcq{d-bI2=TJpI&|go+a7P{4k*#D$f^elF;>fs;@Is+d7p#@|UBbw# zHFdJW-f$SKr z6R$#f8K6i;aR|H3^<&b4x!gp^FV~{W<`|paCHd^w9GO!-m~8nfMH^I0U@U#qYV}qN zJ^v!xBk_yc6Ke(V_A*9*JJA=I{u>PTArL}K4%o$KT49bjh2CdOcAnJWh*~72JAc67 zNd&T-Adnl`Zc6NaI(KA)UN9eCyhNlbq&0*OR-V9CPUCkpHvv^x$W7#@-<~-+{pQKj OC-+ZW#%ajZsP#Ymiq4w= literal 0 HcmV?d00001 diff --git a/contrib/site-packages/Pyro4/nsc.py b/contrib/site-packages/Pyro4/nsc.py new file mode 100644 index 00000000..8d063520 --- /dev/null +++ b/contrib/site-packages/Pyro4/nsc.py @@ -0,0 +1,102 @@ +""" +Name server control tool. + +Pyro - Python Remote Objects. Copyright by Irmen de Jong (irmen@razorvine.net). +""" + +import sys +from Pyro4 import naming, errors + +if sys.version_info<(3, 0): + input=raw_input + + +def handleCommand(nameserver, options, args): + def printListResult(resultdict, title=""): + print("--------START LIST %s" % title) + for name, uri in sorted(resultdict.items()): + print("%s --> %s" % (name, uri)) + print("--------END LIST %s" % title) + + def cmd_ping(): + nameserver.ping() + print("Name server ping ok.") + + def cmd_listprefix(): + if len(args)==1: + printListResult(nameserver.list()) + else: + printListResult(nameserver.list(prefix=args[1]), "- prefix '%s'" % args[1]) + + def cmd_listregex(): + if len(args)!=2: + raise SystemExit("requires one argument: pattern") + printListResult(nameserver.list(regex=args[1]), "- regex '%s'" % args[1]) + + def cmd_register(): + if len(args)!=3: + raise SystemExit("requires two arguments: name uri") + nameserver.register(args[1], args[2], safe=True) + print("Registered %s" % args[1]) + + def cmd_remove(): + count=nameserver.remove(args[1]) + if count>0: + print("Removed %s" % args[1]) + else: + print("Nothing removed") + + def cmd_removeregex(): + if len(args)!=2: + raise SystemExit("requires one argument: pattern") + sure=input("Potentially removing lots of items from the Name server. Are you sure (y/n)?").strip() + if sure in ('y', 'Y'): + count=nameserver.remove(regex=args[1]) + print("%d items removed." % count) + + commands={ + "ping": cmd_ping, + "list": cmd_listprefix, + "listmatching": cmd_listregex, + "register": cmd_register, + "remove": cmd_remove, + "removematching": cmd_removeregex + } + try: + commands[args[0]]() + except Exception: + xt,xv,tb=sys.exc_info() + print("Error: %s - %s" % (xt.__name__,xv)) + + +def main(args): + from optparse import OptionParser + usage = "usage: %prog [options] command [arguments]\nCommand is one of: " \ + "register remove removematching list listmatching ping" + parser = OptionParser(usage=usage) + parser.add_option("-n", "--host", dest="host", help="hostname of the NS") + parser.add_option("-p", "--port", dest="port", type="int", + help="port of the NS (or bc-port if host isn't specified)") + parser.add_option("-u","--unixsocket", help="Unix domain socket name of the NS") + parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="verbose output") + options, args = parser.parse_args(args) + if not args or args[0] not in ("register", "remove", "removematching", "list", "listmatching", "ping"): + parser.error("invalid or missing command") + if options.verbose: + print("Locating name server...") + if options.unixsocket: + options.host="./u:"+options.unixsocket + try: + nameserver=naming.locateNS(options.host, options.port) + except errors.PyroError: + x=sys.exc_info()[1] + print("Failed to locate the name server: %s" % x) + return + if options.verbose: + print("Name server found: %s" % nameserver._pyroUri) + handleCommand(nameserver, options, args) + if options.verbose: + print("Done.") + +if __name__=="__main__": + main(sys.argv[1:]) diff --git a/contrib/site-packages/Pyro4/nsc.pyo b/contrib/site-packages/Pyro4/nsc.pyo new file mode 100644 index 0000000000000000000000000000000000000000..cb3878bbf6e1d62a5a1e326b36e7ee5d2652444b GIT binary patch literal 5439 zcmc&&-EJGl6+S~!lt{^zBH3~jJMq|wV!8lHv<=)qC{hQu6DJTDVdd1R6PKX4Lun;) zmzf#LVjl0gw2=P9nx%~M5+dlYE9HqY}9kW^cpfzi&` zLt}L}OTEqd+Gkn6QLS!`ZKmsbYwY*3R7Ylzd85DD**9J98d^WdhGU!T?s>g4)?e6x zNp)=Wmsz^2YY7+c+s^MYdzhrAks5!uQBD4XU#nKYyuOSi?F^E1SF*<1%({fJN$^mY zbC>YU;NiUcH*S>-lf{$*0j1z4^uV6TR+S1vTqV*Xy5fY2{qIN3DmGR%gfT zGV4w3`6Lr>2Cl~A`zjJ^+^Fy4r0b>DC%$hy*FrOl=VKq)q=pws;;=euW<%fXXWdS} zxtZ+PjvY5YvuOSzvj?sjl(o^DNSePkP^&C;&7U=X(d;KX&7r6>mmF48A9lJ2on6B{ zZT3+!b=?M9EMlUeO;W#^I3LL>S>)!BkgBMgYDIN9&p(BSK@cQf0d+tQDURdw$5`>U zAklJH(B%AW8$$PWcF?GCf7vxl!fr*x2r&@hc3v<7ywmO=Zu6R06C584sp{Fb`8;LY zEo7i?*f}%}yJqfLr&k4f1qGjIpi0k)pn+(Ornndix|bJLPYWnAVmw(Z5fqpxhSv0w zBgcGD*P)>Ath+lw$5>DLCKU+#IIkMYRAZ@#I(FAZOn4FNtyEr|B3;Im3?r{AEPM?T z$D1OxtX2wBEtR;0RFvV-p|nmi3RjZ0m>PH5`{A;d8$^zgyG&p zV$JuX1ouv7snLLD1fupEdf4&a*i^2eHM{0Wz;_GiQP&~}fHSHQZ%Xd(_SoShQ{>%-<}0qjbAq@Wl&G!i^o(eHWOkuE#+n#xD>p+9AmY2{ z=Vai}K)OA#BU6Krb-CVe8E|xghoC_h8aVc zM7TlGiI_1c{y~XsbE53v4YrFgz@-Vr^6d$PA?|}??qrYUP*8KMovqC8G3DAI4z$a+ zBM3s7$LVIH)W2Eyc=-W7-oj5%%E4Pv3u-Bl#>58@@C=&#r~-ST3vdYBi&J<$0o)25 zl*x7kd$=}1jc0{VPCz{oj1zHqm+ueq-vBPQO16;PWZ;GY<~xY4Hej z&t?Pd_l!PgARGEqYxFo9X*WWqHg2YOe+3#O8cX_{Xc_syQ|ub8$N6CS0XM=C-au9K zE|MCb?YXPN#Uve$g74wHO@HCw?F3^&qnjnQP&JT+$izM z99W-u#~gLrN!rV#&~7u2wc9l&UtxaF{0C=^y{~{BuPWlBimq_-HYe9PxrIbV>1KmL zCygC6mFUQa*P*=pPM|fhK(CvZi=#E(+#VL#HxI%}88!DI5>gB0#nRRC)tTk;^6c_l zQO!>Os)Zu_#Y^g{>azMWE{y5}Jnqksfb2kVeWC#v4^bNI3p{l=hO`^wZmUW zv;cd98yX|pK%`OOK;*PgPlQixN{JU9HItXm7&dc^|!g2KSu~H3ar1KxK{3YHCGj#1g9(OZGZ-Y zSqyw=^4H`-YQDOXOeO4wi*qLPI=&wDR9u>ReJ^uf_$oGeW-8e;{h?!;WQjN$pj_eE z?J&kLA7h9ZO~#mJ{CF5bFhoJ_=NN)(n6X25UCXTA>DEP>^fXU^9j14@c0vepf*vXucuLYg1kP-8p z^O-em53fh~CU52VUgmk7jXYRhSniAmqxaE5yw>P8=VUDA`XoK<^pjX)dxOL|wp9~W zx`ygaM4q07f@TQK8jXg8M~&uaBiJK$cRuSReRv-}gRMg!Bfy>O#XKuPc8eXr{1Z$M zzDF!IY@Qy%NpIBHM-lTs(Hc3kz~Qpd*XdlA!Wty7pSJ>1BS9vUun;a3cgRW|HCVzR zbmF)j)LDvB+vXTb)K9+CMC__b?b(q=Y?2~8Vcf9g)^^026EW+FA>klfj7+r7awg7T zJ@6?h``RW^9g-RoWUv^#s;J4W7zA01jo1zZ3&Z0c1+#!0U7Q^#VK!FW$2{vtJhxg> z<-(%6p{|4WSJiEh{WZucY7WneDxzjd$}=c0N-pfY&EvG&qBr7toP7kVG4qI8x7%^n zZMOwAIS2V#A#ew9xh3?SKXn9~%m&7GUZ4Qok2vxZY$G;0(H$gm+i|XkXZT6(;y|3k fDt~f|1iuy`(MZT%jd_c%6{KW{)JS?1$CA^3uoD5u{U{1kS}f3+6}^*7-EAkT6VMz z5RgX{D-9S_o-Y#<%wd^gLmpPxQ$%-(L;@JOMU}|o8*G+ zJMri|Dv$9*<@d#uEglMtsi`uikT&~A7)5FtqVA}&oxXD=TTc3#ZI5rnZP2?kYW?Tv zC*~Vw{N$Zfa1rOj?$~LP=}ZTzT8z6(wK2}SXpEAXBUydXQ@!o*@<93^ Gm3;%avSt(j literal 0 HcmV?d00001 diff --git a/contrib/site-packages/Pyro4/socketserver/multiplexserver.py b/contrib/site-packages/Pyro4/socketserver/multiplexserver.py new file mode 100644 index 00000000..20730187 --- /dev/null +++ b/contrib/site-packages/Pyro4/socketserver/multiplexserver.py @@ -0,0 +1,223 @@ +""" +Socket server based on socket multiplexing. Doesn't use threads. + +Pyro - Python Remote Objects. Copyright by Irmen de Jong (irmen@razorvine.net). +""" + +import socket, select, sys, logging, os +from Pyro4 import socketutil, errors, util +import Pyro4 + +log=logging.getLogger("Pyro4.socketserver.multiplexed") + + +class MultiplexedSocketServerBase(object): + """base class for multiplexed transport server for socket connections""" + def init(self, daemon, host, port, unixsocket=None): + log.info("starting multiplexed socketserver") + self.sock=None + bind_location=unixsocket if unixsocket else (host, port) + self.sock=socketutil.createSocket(bind=bind_location, reuseaddr=Pyro4.config.SOCK_REUSE, timeout=Pyro4.config.COMMTIMEOUT, noinherit=True) + self.clients=set() + self.daemon=daemon + sockaddr=self.sock.getsockname() + if sockaddr[0].startswith("127."): + if host is None or host.lower()!="localhost" and not host.startswith("127."): + log.warning("weird DNS setup: %s resolves to localhost (127.x.x.x)", host) + if unixsocket: + self.locationStr="./u:"+unixsocket + else: + host=host or sockaddr[0] + port=port or sockaddr[1] + if ":" in host: # ipv6 + self.locationStr="[%s]:%d" % (host, port) + else: + self.locationStr="%s:%d" % (host, port) + + def __repr__(self): + return "<%s on %s, %d connections>" % (self.__class__.__name__, self.locationStr, len(self.clients)) + + def __del__(self): + if self.sock is not None: + self.sock.close() + self.sock=None + + def events(self, eventsockets): + """used for external event loops: handle events that occur on one of the sockets of this server""" + for s in eventsockets: + if s is self.sock: + # server socket, means new connection + conn=self._handleConnection(self.sock) + if conn: + self.clients.add(conn) + else: + # must be client socket, means remote call + active = self.handleRequest(s) + if not active: + s.close() + self.clients.discard(s) + + def _handleConnection(self, sock): + try: + if sock is None: + return + csock, caddr = sock.accept() + if Pyro4.config.COMMTIMEOUT: + csock.settimeout(Pyro4.config.COMMTIMEOUT) + except socket.error: + x=sys.exc_info()[1] + err=getattr(x, "errno", x.args[0]) + if err in socketutil.ERRNO_RETRIES: + # just ignore this error for now and continue + log.warning("accept() failed errno=%d, shouldn't happen", err) + return None + if err in socketutil.ERRNO_BADF or err in socketutil.ERRNO_ENOTSOCK: + # our server socket got destroyed + raise errors.ConnectionClosedError("server socket closed") + raise + try: + conn=socketutil.SocketConnection(csock) + if self.daemon._handshake(conn): + return conn + except: # catch all errors, otherwise the event loop could terminate + ex_t, ex_v, ex_tb = sys.exc_info() + tb = util.formatTraceback(ex_t, ex_v, ex_tb) + log.warning("error during connect/handshake: %s; %s", ex_v, "\n".join(tb)) + try: + csock.shutdown(socket.SHUT_RDWR) + except (OSError, socket.error): + pass + csock.close() + return None + + def close(self): + log.debug("closing socketserver") + if self.sock: + sockname=None + try: + sockname=self.sock.getsockname() + except socket.error: + pass + self.sock.close() + if type(sockname) is str: + # it was a Unix domain socket, remove it from the filesystem + if os.path.exists(sockname): + os.remove(sockname) + self.sock=None + for c in self.clients: + try: + c.close() + except Exception: + pass + self.clients=set() + + @property + def sockets(self): + socks=[self.sock] + socks.extend(self.clients) + return socks + + def wakeup(self): + """bit of a hack to trigger a blocking server to get out of the loop, useful at clean shutdowns""" + socketutil.triggerSocket(self.sock) + + def handleRequest(self, conn): + """Handles a single connection request event and returns if the connection is still active""" + try: + self.daemon.handleRequest(conn) + return True + except (socket.error, errors.ConnectionClosedError, errors.SecurityError): + # client went away or caused a security error. + # close the connection silently. + return False + except: + # other error occurred, close the connection, but also log a warning + ex_t, ex_v, ex_tb = sys.exc_info() + tb = util.formatTraceback(ex_t, ex_v, ex_tb) + log.warning("error during handleRequest: %s; %s", ex_v, "\n".join(tb)) + return False + + +class SocketServer_Poll(MultiplexedSocketServerBase): + """transport server for socket connections, poll loop multiplex version.""" + + def loop(self, loopCondition=lambda: True): + log.debug("enter poll-based requestloop") + poll=select.poll() + try: + fileno2connection={} # map fd to original connection object + rlist=list(self.clients)+[self.sock] + for r in rlist: + poll.register(r.fileno(), select.POLLIN | select.POLLPRI) + fileno2connection[r.fileno()]=r + while loopCondition(): + polls=poll.poll(1000*Pyro4.config.POLLTIMEOUT) + for (fd, mask) in polls: + conn=fileno2connection[fd] + if conn is self.sock: + conn=self._handleConnection(self.sock) + if conn: + poll.register(conn.fileno(), select.POLLIN | select.POLLPRI) + fileno2connection[conn.fileno()]=conn + self.clients.add(conn) + else: + active = self.handleRequest(conn) + if not active: + try: + fn=conn.fileno() + except socket.error: + pass + else: + conn.close() + self.clients.discard(conn) + if fn in fileno2connection: + poll.unregister(fn) + del fileno2connection[fn] + except KeyboardInterrupt: + log.debug("stopping on break signal") + pass + finally: + if hasattr(poll, "close"): + poll.close() + log.debug("exit poll-based requestloop") + + +class SocketServer_Select(MultiplexedSocketServerBase): + """transport server for socket connections, select loop version.""" + + def loop(self, loopCondition=lambda: True): + log.debug("entering select-based requestloop") + while loopCondition(): + try: + rlist=list(self.clients) + rlist.append(self.sock) + try: + rlist, _, _=select.select(rlist, [], [], Pyro4.config.POLLTIMEOUT) + except select.error: + if loopCondition(): + raise + else: + # swallow the select error if the loopcondition is no longer true, and exit loop + # this can occur if we are shutting down and the socket is no longer valid + break + if self.sock in rlist: + try: + rlist.remove(self.sock) + except ValueError: + pass # this can occur when closing down, even when we just tested for presence in the list + conn=self._handleConnection(self.sock) + if conn: + self.clients.add(conn) + for conn in rlist: + # no need to remove conn from rlist, because no more processing is done after this + if conn in self.clients: + active = self.handleRequest(conn) + if not active: + conn.close() + self.clients.discard(conn) + except socket.timeout: + pass # just continue the loop on a timeout + except KeyboardInterrupt: + log.debug("stopping on break signal") + break + log.debug("exit select-based requestloop") diff --git a/contrib/site-packages/Pyro4/socketserver/multiplexserver.pyo b/contrib/site-packages/Pyro4/socketserver/multiplexserver.pyo new file mode 100644 index 0000000000000000000000000000000000000000..048c60f11570ea720f3816024a251ddf75f4073a GIT binary patch literal 9764 zcmds7&vP6{74F$tt+cXa$-m^-am*yd$vUy6{34-@<4PFMbouiyLL_ukn3=jia;fB&`LQ2CFGzpvpkzef=&wSdx5VL`PVwSbDP7t}(bS9jHd z+p8DVLa|pLPzwW6cU7yT)&|wWpjsPJ3qxwHtQN}3$;T?Gg_6=m)f`q~QQgNh`&2lf z^oUx*WF_^!Qumem{=%pV2bCU`)}emueifE`t;7A+F|N2;*~g9i9K~=oX{_qZGdjJi zQ*SXaI`op*Gq!oH)5@ZDOK(K+@}xJDXcIr3c^#v@Y$eq}XeNh;Z*Hb={LED-@NAe1m9=!TsFjs)tJ5?|O;$veZRR8@T*FI~wxiw3WPj;8%pOITYrU=L z(C)b^TfT@L_s_V1eiw0>6%-n03^}93iQ)(x^*)XOr&ypgd4#T1N<1C54DxhX8Ol!w zXQ}bxARF`Jlk-^^;n}d~sPgl};Ix#!*JuUCcuPs@?bv~rr9o`kN!mXtcE}H@k;E~C z9VM}83}Bc+TxDF-xXhalq^ML=G!%<)LZvp9s=I2lsM04`#$~lxP@Aq=DyXdi1<5wC zIkdSZ|KAYdEA<-TZ04^9Xu#7Y8UfbiH3t&ibb+#-!NAXFMNNj&cPD zReY0O+F$gh+>*_;#~dzUpE|Y$tHe4BYPF=&mz)qw?QMS|AK{P+_dnE%wN@)*5bB(H zoTANwG=p92I;Wi~C@Q)b#bL(NNHxqR2*Whv`DD?WPCA)9iIXT^(P@-Lc%V!X#q%$` zGAYkmNh4^jBqlT8Li0TxrJ*-7SB1T1o%T8Jl<`t+lGa^qye#qhy}b%Y*xq1 zi)oN{r`|}h+IYrry%2q18^kF!{<UExBz3vm2t-k5lKZ2VI*W#aG4cDHLERVciPjx%3*H58txO?UFrN>*5 z-WgP^=t2NnSwTVUEo68fZ=ec|9 zm<+wLf_{xU4N8Ki?&1=OypPJp7QMpEosJ?D<5!dmxSw8;dQaWp6@blfMCqoq!zIWq z+MV@HwLU1VMGnEPs1y~D47YGt6fXzH4V;biaPS6xYtUz_u;w;a0~ij)zv_)lr*Y8o z^j*LWz%yx^bKXi2hb?W}0IDlN<|U0rC#8-91ifSljXL+e(0txTCMQsHf-!QG*#XQ| zx1&w>J+x@Ks6609z{U3P^?FB}%zqj$2?mCdX#{B~CL{(Scq=Q-aX^sV@*omO=6-J%0!a*R>ggb_E*y;^^1651(<`+PM?j`CCEeWaumkfi! zJ*W)M`Zl-LSdE>7tJE)pi zJGwjSJ%@(>9?aiSx4NfgL814f?Qh4veL=IkcIRKp8_=p`ZqE9LYW+dZGO&btL2bM!Gw+8Vt`IuA z65t@T?sxG=I?NcPol6)y#&=d1obO|NgaghO@OtJqisL# zhpe(uIJ~BD84m^Z3-AOw1QsRE(P|u#9rq70%v_wAjzi11Ihqw1O`QTWdr3|LP`Qew zTp-m!6-FSK1`m~2Um(7iu4|w zYNx${nhkCWLgo3SPsx6Qn=aUfXdGp1p#}yEjb<)^)k!ABc`jX2dt%w5_E-itykb@o<*+tlQ^N_iU z&<$o5{AXD(p|S(;sI2}46rvwhjj$rhy4Ky(`rZgyM*FX@*SA@SZNAQ0pI7)7aa-ZJ zSSL;I>tA8fGs^G$Shn#P?MaL~zmpWG=Kk^BMLY(ULwIuAc zkRuTLN3fYRkS}0#>_5TcDHhzL|2zx&bL&oqP-~}28>yhKj~u=7Z?oWOY>Px{Uw_8N zcd;sNT;x(hBs6lQ?3M@0W97%oN5)F!q6Co^K&paD^&3g6)xazAL;avWn8#3P^hQGCTr@)@sPWW<~h&H6{q`o}Wvc;19C6hWu?Saqevj=QxF zu{hc80o6Pxo*{xq;M=!2_91Rn)(9yfPPJoi?&I7{)!TI*QE1_$T+u%Ds+}9t*Vf-t zwOV`<<4?%of8pTy5xVjUd37y>^p7&=MidXpr~i`e+sP4~-^$;R^OX2r(00z8fMy_# z0j@yHILnVU^dsU~$PA)?xKO4?(92{=yGf>Gy?gk#7l_2<3y@Au3x=+c}|d&)f@ih z?bsOVgseSU^NhTO(r3_UJ=kux?w`Q}LFi80pN*XVP2F8gkgvK-0i~U`P2{fxhEa1x zaGUZIzeiyqqz&=G1FRjer3B{ef?wEn#S%iK+^3|vQn{z>4eSOR3`GuWfmxLp)PIg6 z$&+{+b-sR-x2J_Y3)o!3BFXkg99OwO{9~wl2q;HTJA_OaL(LLe#*v+8Zj8PX5kX-X zD{W9ym%=9$7B1_{2mz#&)40q};f{CtvW@kgXZexS zouX?aqzg#GyV2L|aB5EXSxF8`T74P8CI*u)!pxu+cMy3O@cty=7hjBr1%ZD|;-9;) z5198vPU1!{xdDe~Kf!*c@~k9~9;>jxTZ<21q!AhB4?A-r_QSGtvR?r3`PVKU(E@`9 z`lHX7`q=O}!_1v4`KJQgi=O0jc?0fCy4x(TnRak%zd?%=LNs%rKj5^p{<*?$t+qan zbxBT_q0QU$G17Ecmb%INT+}xHyn&~;aMgBsA~+!%yV!5Z;#X|!Nu+<+x%^kZVoiiD z#p?CLuUD96+;%Gd5^vci{z0;_q0Q4^VQSku_^Z5?WX(K#GYokvXmzy2EdS{%n9>jr zVM89OpudJ" % (self.__class__.__name__, self.locationStr, + self.jobqueue.workercount, self.jobqueue.jobcount) + + def loop(self, loopCondition=lambda: True): + log.debug("threadpool server requestloop") + while (self.sock is not None) and loopCondition(): + try: + self.events([self.sock]) + except socket.error: + x=sys.exc_info()[1] + err=getattr(x, "errno", x.args[0]) + if not loopCondition(): + # swallow the socket error if loop terminates anyway + # this can occur if we are asked to shutdown, socket can be invalid then + break + if err in socketutil.ERRNO_RETRIES: + continue + else: + raise + except KeyboardInterrupt: + log.debug("stopping on break signal") + break + log.debug("threadpool server exits requestloop") + + def events(self, eventsockets): + """used for external event loops: handle events that occur on one of the sockets of this server""" + # we only react on events on our own server socket. + # all other (client) sockets are owned by their individual threads. + assert self.sock in eventsockets + try: + csock, caddr=self.sock.accept() + log.debug("connected %s", caddr) + if Pyro4.config.COMMTIMEOUT: + csock.settimeout(Pyro4.config.COMMTIMEOUT) + self.jobqueue.process(ClientConnectionJob(csock, caddr, self.daemon)) + except socket.timeout: + pass # just continue the loop on a timeout on accept + + def close(self, joinWorkers=True): + log.debug("closing threadpool server") + if self.sock: + sockname=None + try: + sockname=self.sock.getsockname() + except socket.error: + pass + try: + self.sock.close() + if type(sockname) is str: + # it was a Unix domain socket, remove it from the filesystem + if os.path.exists(sockname): + os.remove(sockname) + except Exception: + pass + self.sock=None + self.jobqueue.close() + for worker in self.jobqueue.busy.copy(): + if worker.job is not None: + worker.job.interrupt() # try to break all busy workers + if joinWorkers: + self.jobqueue.drain() + + @property + def sockets(self): + # the server socket is all we care about, all client sockets are running in their own threads + return [self.sock] + + def wakeup(self): + interruptSocket(self._socketaddr) + + +def interruptSocket(address): + """bit of a hack to trigger a blocking server to get out of the loop, useful at clean shutdowns""" + try: + sock=socketutil.createSocket(connect=address, keepalive=False, timeout=None) + socketutil.triggerSocket(sock) + try: + sock.shutdown(socket.SHUT_RDWR) + except (OSError, socket.error): + pass + sock.close() + except socket.error: + pass diff --git a/contrib/site-packages/Pyro4/socketserver/threadpoolserver.pyo b/contrib/site-packages/Pyro4/socketserver/threadpoolserver.pyo new file mode 100644 index 0000000000000000000000000000000000000000..3455e45ff5042cbc7409c71931b3977cd4d7d18a GIT binary patch literal 9070 zcmd5?O>Y~=8J;C6O4NrX%d#BhquC-+L$`^PI4Od-u8~Am5XZ6PF6GEo><~-tNLq^8 zrFVw57}$kdr$8?O`U6^^K@p&r7U(61qKCFXfSz(pe?YIj^;)2Po_Cg{9QOxM){bUo zXJ_Z*ndg1pcU}DV%;ZP^{48jy{HKV&Z$41U{27@@sU4)AiVCXjsl$TWDJYNSqG}h_ zPEqb5Us5}zett~tjP>(nwNvir$JNfb#>Lrd2ei z9(rnLMnz?%PpDnIa$J3))I+6yyfdpJv~#tULG7H1CRx}y$>G(iQ<#AHJCey-+C0$K zH#)ncGrtiU9r`k{7JsH5yN}b2_`0wq+D`mZh07&mhYoy|&`+yVY26bLZ6UDYRIViL4Fg@m|_!qFR3V z=C>c9|8vM_EHpL+X@Q%{bq5*>k^vNok^v4%k}0d8Rvp7*<`BuGkDsmZ0BhO|Gwr9l z!?k^ECH*jo{IK2jGkv?OjWzPL-4CsQ820=y%i=rQkGdK5#*eM>+wm@Tt`lMp_?~77 zZ{)@GS$lvXRFSbtwLEptBg*b6RWGP}_(i6u?%@|1OhN%gob-kaJ&={3L)qbJxJZ^W zYjRm#E{vi~mOcux>q(Vww@iSiT|3Rjvdpd@*)v!179I@7ai2qUiTMd~t7*rsw$o#Oa!vDy&~vSebbU+RR-gK$st zN#J4iI38Aqc*p^QXD2X(dOc2JTd$u%-Vj#GoA)YSlZ(QORkFH&LdLBwV1b2f71TqJ z26IIo-|P$Z@k4cVk1LY>M)l4}qcPPgOJ(yKv4G->Wppzdba1wF(a+y2wpZ-o>E&x(oXjT8If*u_W~}JC64w1 z?Y2IKBVuGL>uSM(z(GGC&IDL)%cM_vZy<;NG;R-jTWhBvx^8h5G)YS-N z*+LK=aEnI~bliY?{b`KOJc&ekr@U#e2;ddHGu~3+lvncRP&ZQ)AaYW?1c*OHruU3& zGd6R8MNefhLy#DNT-+@wTT!h^@*OHc zb^zwR5&?_5f^bw#3!V~B-N35Xs@(n!^BtrPAURRxI9lcYntf2=c>wAM1Qid{q(H}^ zfQ()rG!qB}1fu}U^o*QNebmJH+)V`$a;k-I#0IeOZkip2b}I{;x`D$Vkb^9V=5ClJ zAWmuBO5-FThNPEw!ZyIc)f8|@3w>0#{mdPQKGZaAf_>3pfL7+>-s3%@q^@D*%UZRPrP^YFrvk^a<)RAtjp+ z<`Q`X^2rJpo|I>%B%M}(N$UjB6rsHUjcD4w{W*Ha`zM6aK4ZVAB2SUH@H;C& zgMrO0D<>OTsDY04ZR*E~1>(9a?Dr+h3#M;H{dNkQLWxw{tOsi!Y+b&(zA5_{$1x$e zQ6)n;#u3v+>qeF^`&}EQcaz{G3OEnw;oa+7_2ANWApIF_8!%M|KvnQ8n{a_QYr;AK z_d+W6LxZtqPOoT7KErsVSlfKRUfW!|vc45O&*r2{W3z74L662uP&%r_#U&Xm{&^05 z5eX5v2+mprW6dIe4!kw)Rk=91eNAA{>vdUzdi^ksx@{KOpk9yCX1yM8y8?D5;0|6! zZg}P-Hv(!QAmeeL4VkN4Dtnc&%6w(oQaL!qQxhRx-xAqP>Dk0B`5}?K{eU1tMjw}$ z!n&WpC4eq-Dih+;sZ4knqB7w#umPNmvWQOj2o-S~P&uje6upK|;6k7X2SXeOI2aJ7 z;wfCh8*LUQrUQG~_Z7gKzO%K*w7q%pn;rnB;KQ|`=vrx#GKpE@K!}BuK2Stt4jL8g zDo(r{W=Ks~k?ea+lS*!HAb9SBYD5=d!uMphKm+&yq6ibt^H>)&n*tY<*eALIHHZrZ zlg64EsS!tNi990$EBjh3m{JE5%KU|SM<-aJgQ9YG2aTsu=J?@Y$>=EOZhx)FL>TtJ z=yoAMwk8C&VZM!RfR~h6`Ul56y?kGt$`zy5h?7VN zEz@xRM8wPSj1P6%wQ?tciQL!Fdog-driA2`bFZ(+J<;d;DLj<#qx!Cnv&g@+Q3I50 zw?p#|?lo!q4!j=F^Psg~We-REcUC%FS?!(|3by3o^M+&iaM|2CzZ@B2YuPZz6Sf@X zKs3J0P)Ai^2eIxP8zq5L5iD(_i4LeloF^lG2pnIl-Kh*Hu!L%br#SamDXGcdd#)l z8rTkmH$1(rwhL(ENa7k!n2z9#{zz`n4h>py2+omjrrpF=dG8+jtNs2Erl2sjsN)*}O>C2z}*+Q)7r9<;}tb&kmQ0XG{3{EHb(8iUsRC4qQBugL!65?UvMHO~Ni4hDzrG zjyU|kW!Q(|$&@^>YAy;eg()zKc4!PN`YU)WSZBiJ3oauOHGl^km`MR6>zrk}lOZmE zAqsIZBEaR99}ZkrlURZE=tD;)M3-cE$|N@mM~$$!&<+n9QFxJYDuYWkd4xC#@`}vp zAILmn&yT*4!y@*aN9#)oEFskKWk0;(%yL2PNx%u>eh%VZY&)YGzZE}JY&(u3to;&9 zCXfP~2#JG~K4(cS(Fa8tePC?AP}g{7;j2zb*aL?2*9!gK{+s%lVx|OUWAA^J-UZ1n zs_I?C-5a>J^I155M$R`+%7(-Whk>BJ7|H{1^cTsEU{}29EM1o z3hq{z?HOn1*w1{g z`RO7dK%e-rjDbM-!5+Z{MzAh>Pml@!w_ihPo12k-JRlJt4ymu>nPceU+Z~YM^g=qhkPg{pq<|G%$B4ir$}kY1#uz%8qrmWhl_LA_%eDpVB$mPEG=7Cx+n@hGj4oj0>(;4%Au3`S_BfH{_)#&#Bf)R$aQa zEOZN_Ki^kHUy*PJ9*B4QD}~=}Fmg5{-VGjKknYeuUe#9zWT+;BtwR@tUT+S^s>rAR zLwUhlcf-RGkk%#qeXVC9}(7|Gly6{ z(de3?oKye9wXT15sJOVQ_C`}_>0$DCVP&cVm`j^;!9(#fshQ@nT&yYDrS^MTZ z_&W^KP|he9F6Zo6l-Wr*&rQ&Y6u~c>* zUQYF*yU(FIc$LWoCL2sPnFLI3F(C(vqwC`06DWe@J36zypu>9^k|EqBHv*y{cn!n4 zf=j{#WuSR_u~M#-W=~g6Og}L@CfMfAQ#Hx%=MrE*;J`TyGRP%@@h4#y@oj{6p=2FA zdJD|Ca5y3hgak!C20uhu0@;ERi5BDo$2HAsiy%S9s!xC>s|+^DNy($ryItGO5S`{hadVdi#b?ev zhW$mAgFaNS!bH>)iNt-Sq?~a75#d;AVW@OnsRx)F1$;?tB&BR#^cZTGUyEDs~ zS&CF7qp?-TJ<~KrnxqYypy?kekOXK@6fF>>7uuplQ6w!86ewC0=|5@E073oxWe%7j}e z>`=2il$%jYJ<7?c<%|kHs@xrFsaMVJRDPG_(PpP|+tgB@a@wTL?ti__DZ@>(*2SiPzzY-L1~Pocd&72816x}bVwQ;R&N-}@KxHPQ(8RU z(&CX#ExM#dp{2!~qR_ zM9K%%wEsnwbKk9&yH$8QYZ%7*?G%NM2nzim<6ctEQSR<_gRmoV9#hrlRqe8xy{u|4 zsM!}(?L{^FqB7jeYH3I<4XfD^<-VwvMkOsI=|WbyBdUBw9N^DJ+{4&^+-GAGyL#lSZ$w-&^%IWf27R^n*DG)IE^dgv`I#^&6*ITqGj z-*jB_J%PVq<~=^X7}~dkaMknO0pE>J4Rm||jh|v3v_{4Ik&FbskHNgaAE^eB>lDH; z2;&~qOnEgoXvDf;k2I~-8opPt*;*IpP`qTX+Ho5)w>(WvFM;iSCG3>o(qYNVC-%aKcjB!hBd(1ytQb@X1wT{ zW0U5p8%6+@=|yHRFNF)ZWUfr}ECv>C8%wJ%^MH~}{yyAsXeA@J4 z6O}=A)s19?*1l=SI4~E0IkwM9lR`bn)S~NS3_W==j>A@!z>jUuk4)R_AMEEWSR@J~ z&v8T7F)`$v%Mrc$>cw=-)3PwzcQ_)tv<@=RFKfjp=y5^M4!|ij;1NwEJLt6&YKM_r z_m}&>aWT)c5){Oq#_B^FI2KEF)o$ccC`^P&y&y)ibzUwdNwkdInV=q@ zsRk9hdS=|43+-_I%#{$(zZHbb(HVWW2IkX}GdJ|1MQ6?rJaeY%&7G;s?c-DA#qLve zyRvLAxa=7Q7tZKg%S$_ehP*kl>|~S+c#c@8I(m#ABgapsLQLe9;pJdH`Y$9O#-BkV z1$2BRV{$BN#QXvRsh8ZI0C3Oc4I@NNglxNl545{Zv;f%VIC9s#k)U(9z5}r}b z+_e8Om5Xy~32YpM`TGilkUfR-N1tgKL5A)cQ^p zx2rojJ^IQQ^ynQj`ZqW_ie~Vc=A<3M`BQq2RpFX(RWsO5DNi{l`doUQEH`e1K+fbI zCFe>E*>sBr%&~cMJ!qJ&Z_ia-spYxaVbJg$jjuq|TVAzlHX;|;4%!-{;UZ2ky%#N4 z5a>J4o(cYQ{%R1^=h!+~Me}0mG2#%Un#~m+^@ zyS8KI<3&;iAFTuzWq-Uq=~Ql%(1H?fw^vVa>iLVDE))>mcRk(cw0YtFx(lbxvn;7% zfQgb>Sqwns1xj;n6k~3oK}G&YD?&)HdH^?j-dh-$vQab2MDn^MwABa3q8zLjz?ExXks)W#;od)LWqk%%&a<$X%6#0F2!4h3co^(Ug{F@)KJkE1;K%x zSjksZ3?UdDvB>lWX(TKPjS`yWVN)u40!bsGR;zZk;mR@D;X(rpw&hp`yh%1Eu{(GE znSqF~IXfV~lD{x;p1*l=NZcAU6@D-5ZstohPDJf1&$_T1owXD^=B$>6j2iMFn|a3M0$qmfV^P=KZoFgAIk z$ux_3LJ3H%6z`eHN^;2rsQRMCt(d1#DhD8}|6rboQ64kM%H)|X>x{N1{UB;8`dyTBI6S+=fkI|$l1k?z7 z)REwlYcVaMEJqT?pa{l?0;EG-pof`n*SW07lS4!4qlDOK%S9A@$)bDNGADV22H1uOD)jzFeBH zybBWu;%VftcoXHcEn=%h$y_eH+~Cox*3dPf49NuPWhVil7%2!*q5Q6|F0Z*)FDElv zk8(6ZmF+cB;NhWPOu^kMCl+}qbJEz4L})D9p7V542q6)aJ3M2oeNV+%6~5D0FA&7IFi>m&58ew5g?rVZ?jWyqB6O+q zvD7TZWO#;o1@lc#uUvx6ht7~lAyIywG9UOB7&Q8WeW%skQr(YQ12=jJ78V?CI~X+> zc#paxq9Ib)oyfx9z&1M7olXVqaJfT;9Yze>6CQW5DMn88Fz#u$TH2$WF5Y978enS3 zA_vf5_8EMp(QfT!gS)avRND!^z z#u39kSUF5Pp^AQ^HV0W5gmi8_h5$Vu1~pV_nMg~SW1^F~Yjx06YNVj3x|*u0mi7ZP zYSinHpN0Nz8Z#M1G*vJ1`=LyWXkKyaG+}U*(Dq>TF!R(PL`U$0*o5$amE3k=da1hp zLcB-?yitus`Gp2zC(n9Pj1tu#>0>ikgs=5~j%P>97#b?esR45eYp+A@^6aYV*s(2L zWxPc@f}#zZK`DpK&{mBTU7@*&@V?cDS0ees|FC(#u zyOwS)5F^n*lPn6i!QI1^$s(y#y@)b`p4Iag2F+L-e7q)ogoxn>!na#mB`k>W1H*|2 z&4uqSe)Fy0|Ng&S_HO<=4*QY&2PHgRKWHp0%D#*FL+J{l^+-gUdex4p`2!lAA{kpLQH5sp)l5FnG0AiPInC@NSm z9F?uYlr>f;S?HpNjgod8R#-(;b=|skF0F^5M}#-KUUz**hFl6zMj^Z9kn*J3)*1E^ z_Uqktg{wn#OUS*9Qi~S252^%}MQ9q4j8nKaIrU0;_?4+bNqQ4)#E7ycJICS3%B06O zGBIvBL0#hIhwvE9BT*e$qdocUgTU9`(FJb5AA(*E@^_cf2C=UXpIydbNalx)-NwNT zTl8g)qF2u7HTI!Ci}q~Y1rc#``$H%Z-blbTdE@(l0;K3-!lnEGL&=+J9i$tD84`FB za8!WQPJ`6K2oFXH-Zdt?3wor`e)6uQx?QSYmuj#m=#1c9TFE4<1DyxokWL9(LdO~Z zYh;384QqeJIl~*q`Y~C;j${eoo@6#T^(d6y>6MRwo;H{A4-YIQsn$!egeh!Z3fL%w z0WcK^v|ythnvFvKg+QR$=y_qy53ZMNv`;NT$^uu)CTw&!v*c9ZZXW>~{fL?-(6pWA z0Dvqsrv&r|QHCW`sG2r^+(*PqtmIvHYY_}4-~jb7fIS3H)IrUgHzSMz`k6m<^CpNV zDB=Bt9>mWN{HNw%z*`2LGHZe3YRZ0v)>D{LY`@~3Hm}(gS~zCR$mQU$X25>Ah#Ly# z5May@cJ{*1)uFMXDY_Gz1fZR<>+9jBXpEU}_+kw==dmPEX4-wkFv6)x_!k;gJH#B= zHw{BHA^EQBxQ-sA7*XM2^#e3jNMp0+K|0Dy!yoGc5+T=3(wZl!7SaLUl+wDBEF)!0 z0H{ghyHT(~1}!QY)+HwRDCPB%XUG-T4dV2arI;%w$15Ht5B%Y&+-&D2u3?_tTsT$KZ*PjE&$E9RsBX zsR)nAwS~GVS_m#eQ%0-9qSa0Z97s!WiBdHTtso*G98gXm*?}%&1atvON1Lc3B|>;~ zNf_|pKuI8B`d_60+7o_)^T|IDtY4IRREjjd)GZA^Zfp#^L+IM8>`GD;{T-#q@XxmB zwla{Rgst0QO^I5t$6$wcO5?Pq>FR?ZA1$IamMAknFoZVd6%B5pOT=W|pTmIeU`6w zy@|bBQW047SzC4xNm!~sMZU`G7Ccfk;#w%o{o@ZOPL^lgJM zhaLs*mZCPQBvflQT8S1%E%&1t;278hyMk!%R?A%~{6Ypzm-Zxjh71d2J)Yf+8sExn z90mvbR)&fZ)}?_21;_AvS+KN^W!RXp4w8@O{mOxR8_&}#$FsTkfN;M@=rhIh3>O1- z)}BjRLw(U4b$X>emm>rYNrz01n#p0c1jjw51VHcZP%$nQ+@#=6Sw$TWcnMG*701R8 zv&klbmcNi~?mRxG8~{;Ub#iKLPQ}OJxyZ)w*<&f)R78^$ZF1`U7`9j4-Kp*hobQ11 z!E5r3?BvR)vuJmxPo6)W)hmTlT^}iG@i)Da&AkqsZO_R*p|Ty$#U{rmJh=64a6p>s zqN$Vvj)bla8Qe~p9z22bHk5@_Re|pbvNH6Q&^4DKMA}sd`Orh?goB}9+f; zP5tx&_9qMit~vVXAd!Nhq#iaqieP+pP=4L_)=Vd;!IhWJi1wko(tvqCWl9s^sNpSW zI#7VXOyOm#nsKFm1_rGSg$lO+gokNo!|KK7DS4CoRgyi@TL))g(kHYzEHxP^Ph5V@ z&aFNLa`XqRn>3ff#{gammgX;L7(|1BE`cB*N?>4CY@dq6TmXRqTs?wD=*@csfOfS; z^rez7YTG@AP-16`$PER~i7kx4rV&NBF`2RMF%lTI(*tIoq_hdv`(f5dTeUrIcOt-L(|w5aqZ1! zwpVqXV*Ui1Q3bOoJy=gLc?XkzB*NI^^|~u!f;K)=z7QY8*o;^u#Xc#DN8G4aU2V8L z!p_unaFoShlx8(?gyPY10dD#g(t=_z<^yfugyWXWSH~xYhsMjJg)2kX$EV6uW7i53 z*Qe6K5>ZDiW!(9Mf8FQtO(I4?R?9+BoS@D^kQ0|lA=QDccVY(723y1+k$EWYh!$0| z*Q{sxKww2~6_$on0!$bDa(Zd=loPBn6AmnMh1?!-OP-qd+algckq@;{Zx?a}jIyiR z#p6SWC6u~Q*hB|0)M|B+Ccm{+IZ*MpM4|q7OaF7XQR<@cRuhDh_^-Z==ji8=sLme9 zSSRt@14-)$G`DW!Nag^vxg#h!V02}UW%{-*g+{m^?RuLZ>U=%W@7j&WjD4A2NN>AQ z+kt21cD3yiy^yV_TXIHv)(>a8WrPZ0P0&)h8^I@<0zje9)K4%4=ced(qQ%E8c4n$% zXbZDwPKD3$I>7ZvF$9@*8dm*(A+y))5(WmUmb*m+gUvF!Bi-s-bXz$|#fzMmwoQ>T z$dbP*9lp$)1EDObJ_u>7MFdM{eqab~82>>s8`hTe=Req0F%$k05e(B~3RyVfn+|A> z=kJw?H$rtzIII@rmX?RrVI#tViXW>L-aW$ zq7FBOfre8z=fc2tDmJ|Anqg`N*W{1ez+5+TN0PBES=T0hsAZT|ZrIAMTDak$`9x@) z$EY3&n0U^mw$=a?z_=+BoHoxtCA@=D&IO$<=IQa;iYIBMR%*pFc7yF^n25_vSlN?! zOgRQYjpfQC!^G%7D@+M3FHMw(t%;%0ks-Lm5oW8bSDCz@$p@HF^bqmgLkh$%&Yug}=~rdCd-Cp#|m8!4{g;n)6T%Be5GN;&0&d+%lMc}!=Aj`$$>c)kD}G#sq^H!C^3}rBmC+<$l6*JIN6RDQ6NQ<= zh|sVNp?%)bBUJ5$Xd6k)`>@7et3OK6 zhf4p_P;qp;aKB4dN0x9n#Cbc_syNuDy(AEvlnmum9-}5DBU2o~M*snf8IK(t0TL=? zU>tQ;MJH)KMTs`9!YowEvsd)fpR8~Ov7)y`yO$w-y#(7!)g4339QAkDW~|$o(Rv-p z+k!jr=zg0=M?k2|p$*P}LJd#WY}^8}_Mh$+n0cpBtF50ZYK-MP+sz{hh2JF{ze_Uc-YuCP<@PEUspv;|H_A%X0Ip}qycNCN{ir z+Q&cUR0sfm2&@r&B<2x@S=>0-E1**`BS0w$j_VkE8OVAQ?e*m*TEPY~-%^f}Z3?94 z{>t#)dvUq^<>qyOvfm^x&FethO_QY|swfVqL2>9;PR^gJR; z4@=t8SI?eKwV$Megrc?{E@cjZfw0$Tu@O6XNk*GuY0xUYbbYF9jZRyFO%tVp1O=aB zueS@H%R>H`K%=0fV6Zy~-7&Yty=8D%6LN2CTwB2gBDZknx5M3)h5Tg;B5;Z6r+I0c zdr4ekn=h?wi8hCpOH0$j_91v`HJQ?(Uw_@QL!x>5ksbc%(|RxW!amkEQ(7V(0wt1N z{q4rCiupcN$=yQWEu2Q>vVfpmMpQ|o%17QnQ0bt8Ja1gN994rjDwnl&>MaymKgWdh z&LV*jgx5mkFY)o0k;oKUnvU1SezbAOl6%Pv(K&1ekF>dExvy0Y@yA!$Z} zsVD?w@sTs9XbF-HWevEkSrNi&p^IrVEkl7p-b8{oUPFbBgV46=VKKEMDwi*yC8nO= z#CV7Vl~=pqHEf78$E4wmZbTSL7lQ`WGrIU1UMxBr#B_1iQC!Mby=Aw$e%eIXBa}Nt zYBuc9Mhw2|B=1Q{3{M5l6*~GVL7hHAi7cgt7{v}==YwGb@uu+rlQN@6IlDgo0nw2} zbI>LwvSMu;Bz4tZq>IRjlA`14JMl=Rld6xLIs%>QEU*B$?Y!DNXTG{HiEt}Ga(%;d z_I08Gg(j6{bX_+{bf3{lAB9r#nj}q{f(%3iUI;;k8|X%4fEe7|Lke@e&D&WKH8U>J@hZ=o4H~@9Fb=H6 z*h%NR4Fq-(HAggM*Z_h@aet(dl`(gb$T13PczU#)9+|GG;P!W3$4xWpS;D5tU?ibdP8fN4jvO!ZE z*1`w37)09ul96QCjYPsPAXGv7iNx$qGOkVXa$AY^+EaalIW9wJI^wJuF>Q&Jn5H!A z6HLUND+0$&KH5x9BiS(PSP{#%_c99&`e$C-cVICpdkJB(kBSR)7aXj-#Je&_VZmp> ztag;{GQ_B*UD}CzqFNQ*iMyqhYXqO@JxJE^RS}3)h4c#C#1+!MuoJuiDD<_E)(oAU z_V=~Msuog_7I!mwDgGG_4As@Nb!5Db4drXo2&|V7QK817&7UTUBerE>0V*?s0%a=LO z=8s0{H#z9A(oqyz>|~LZSY()bv|)a~17$TGh0jN;i?wXh{K5w^D%~J0ve?JE#e@^; zKzQs> zeneXRt$;~`iFg@bVGg>0vOdVQ~Jh#*^7f&1w6gf%Mt5tbxl3w;=#m>3sIiI-hi zDiWi99g#%T`ZyE%NkvWoIW0-JOiElQgKE|KEmUt@Ey+Ff`$fQ9((>G8B5m{#hLke#rj5i=K49AN%{p~6(C-5 zW=X1UL1J=tVtOh_XHmY1eo1O^iGF;1W?p7Ve7qh|EeFs_o80`A(wtN~ki&|Bm;nG@ CkTaJ6 literal 0 HcmV?d00001 diff --git a/contrib/site-packages/Pyro4/test/echoserver.py b/contrib/site-packages/Pyro4/test/echoserver.py new file mode 100644 index 00000000..9ea2a631 --- /dev/null +++ b/contrib/site-packages/Pyro4/test/echoserver.py @@ -0,0 +1,126 @@ +""" +Echo server for test purposes. +This is usually invoked by starting this module as a script: + + :command:`python -m Pyro4.test.echoserver` + +It is also possible to use the :class:`EchoServer` in user code +but that is not terribly useful. + +Pyro - Python Remote Objects. Copyright by Irmen de Jong (irmen@razorvine.net). +""" + +import sys, os, time +from Pyro4 import threadutil +from Pyro4 import naming +import Pyro4 + +__all__=["EchoServer"] + + +class EchoServer(object): + """ + The echo server object that is provided as a Pyro object by this module. + If its :attr:`verbose` attribute is set to ``True``, it will print messages as it receives calls. + """ + verbose=False + must_shutdown=False + + def echo(self, args): + """return the args""" + if self.verbose: + print("%s - echo: %s" % (time.asctime(), args)) + return args + + def error(self): + """generates a simple exception (division by zero)""" + if self.verbose: + print("%s - error: generating exception" % time.asctime()) + return 1//0 # division by zero error + + def shutdown(self): + """called to signal the echo server to shut down""" + if self.verbose: + print("%s - shutting down" % time.asctime()) + self.must_shutdown=True + + +class NameServer(threadutil.Thread): + def __init__(self, hostname): + super(NameServer,self).__init__() + self.setDaemon(1) + self.hostname=hostname + self.started=threadutil.Event() + + def run(self): + self.uri, self.ns_daemon, self.bc_server = naming.startNS(self.hostname) + self.started.set() + self.ns_daemon.requestLoop() + + +def startNameServer(host): + ns=NameServer(host) + ns.start() + ns.started.wait() + return ns + + +def main(args, returnWithoutLooping=False): + from optparse import OptionParser + parser=OptionParser() + parser.add_option("-H","--host", default="localhost", help="hostname to bind server on (default=localhost)") + parser.add_option("-p","--port", type="int", default=0, help="port to bind server on") + parser.add_option("-u","--unixsocket", help="Unix domain socket name to bind server on") + parser.add_option("-n","--naming", action="store_true", default=False, help="register with nameserver") + parser.add_option("-N","--nameserver", action="store_true", default=False, help="also start a nameserver") + parser.add_option("-v","--verbose", action="store_true", default=False, help="verbose output") + parser.add_option("-q","--quiet", action="store_true", default=False, help="don't output anything") + parser.add_option("-k","--key", help="the HMAC key to use") + options,args = parser.parse_args(args) + + if options.verbose: + options.quiet=False + if not options.quiet: + print("Starting Pyro's built-in test echo server.") + if os.name!="java": + Pyro4.config.SERVERTYPE="multiplex" + + hmac = (options.key or "").encode("utf-8") + Pyro4.config.HMAC_KEY=hmac or Pyro4.config.HMAC_KEY + if not options.quiet and Pyro4.config.HMAC_KEY: + print("HMAC_KEY set to: %s" % Pyro4.config.HMAC_KEY) + + nameserver=None + if options.nameserver: + options.naming=True + nameserver=startNameServer(options.host) + + d=Pyro4.Daemon(host=options.host, port=options.port, unixsocket=options.unixsocket) + echo=EchoServer() + echo.verbose=options.verbose + objectName="test.echoserver" + uri=d.register(echo, objectName) + if options.naming: + host,port=None,None + if nameserver is not None: + host,port=nameserver.uri.host, nameserver.uri.port + ns=naming.locateNS(host,port) + ns.register(objectName, uri) + if options.verbose: + print("using name server at %s" % ns._pyroUri) + else: + if options.verbose: + print("not using a name server.") + if not options.quiet: + print("object name: %s" % objectName) + print("echo uri: %s" % uri) + print("echoserver running.") + + if returnWithoutLooping: + return d,echo,uri # for unit testing + else: + d.requestLoop(loopCondition=lambda:not echo.must_shutdown) + d.close() + +if __name__=="__main__": + main(sys.argv[1:]) diff --git a/contrib/site-packages/Pyro4/test/echoserver.pyo b/contrib/site-packages/Pyro4/test/echoserver.pyo new file mode 100644 index 0000000000000000000000000000000000000000..213edeba6c7dda385ea7df034402d4b7cb0f39eb GIT binary patch literal 6168 zcmd5=TXP)66+Sa7t+ng*1=$GOTs-&^uL-*X0!2^(As7>!kcAo0TL8(vzuND!(HCArE=TTZ+mLNmVNEdCWudeWzz7#ZK}nWKVBjPIsUF&ZW=k+CR=T zfBDzn#{;SUYWRH*pZyDpRAdchAZaM0K-N(4^-$KrS-mD}HLcfVG$-SES(}%DbaUFS z%Ua#HvBsRN%}F{hTUhKAD`r79L0Xq@L>`HJvDT3El$d!LHg#a(aNxA04K~(VV$Mj~ zly9)cSuy7%JuN$-lz$e3W+21!lD6dPqd zoUtyNOv_1OjqNt?Z)Y~b|EZlOqtRZJ<-5g>Nu!Ou$hxF-S-urH(u|9AIxs@ZtHQ3#6zTRx!bKEHz*&@R3Hrv2@ zuE5s#*)~{elvum6PL5ZV;dQXX;W8Q&scCLZ9S9R$B`;7nWr;=hNU}K{b(>A@9Cg5e zvJ#td;Y{?&#?TC$?MBg^VzO6eTicGuyH}1)9;GJwX@N7gGdABVldp?%H_J^oH|}<~ znf(iYy*7Xp*JQC>nk1dNY~=gu=*yEaP}1Qe)(_Q-Q{TsD-#}p?O$Z-lC3>84yrbwopdzd@p%!|)LtrLlv74okph`*ssVH#fW0LIZ zV)r(q%-LuqajslhhmLGOTncY3PzR(A(-b!Tf2kL-bAr4G88FaIHW!72eKK;(oa5u zCZoRE<}`Z_MQPl$%y9%LVYy{3Rs5!fvf6}er0 zt^4NkDBD=}#=w@%oas!G!45IPOqj+pO=FpCd(-HGnyj+tdHh;xm4uDpQZS(U@FNV0 z+;vncFVo9X8v_laXrYapUC3Hr!TPJk0GPW+<}kQqRinUYlG# z2BGQ$?afim$5ZS2oJ9KX4!R3qZN}AY)@ORV7_=K`I}-H!@QfICt{;9ZSZM)mK@dR>H+?Rje4W5XY1jZ{@NN~ zSpR4T@8h#qP?(wT|3=+^i?AOmWth9A7|;Uc|W0>fcl z9VlSM^&yq(?5U}?iKfDArxR1gPqFExljWI%2#7Qs{Rau|%v^`yog76MW5mzByK8c% zg458zq{_G&dQ4o7gmg0Uxd0ws#b>kcC?x`1hYB5Uoj|_y<~=iV zLOQSRpf1BxL{(9rli`9k6kQ6pr+OZY*jj8-u?0{P8oK zNkU-nsQ0CLFvacgaZyaZJBCgM%W3{5`BWPZZU9@2OJe$`_$&*;AHRU&NDs7jGRi;| ze+8eV-0MN3u2(57jo%%P+(H3Gpmhau0ChkR5_oBAI07?!MA0)0@zUm6-gxM;)C#-^ zIq)#a+);Ev^KslNKk?Xt(UbCeVNlXgs2*aQnLtfKlrh?Q2X-|p_1{7yOQJYa9kwNBy-lxzT z%z76ky&&mDNuTCC7zZhqj?a5W(o0WJTs}^5MN$MukFD|Caf;_9ec=g;tH&uKNfDZ2 z?`!Dw+|52=XK7Onw~BfTLzH7U~| zy(#bsNcZ}uXXGuJOoH@#G6YJuR2$)b=4JA`z*7#-&ZeBLmT#4RIvPVgZC4Z^ts+5@ zgSt|mm-26cdrCQ66n8;8FX}eRF!uo1x2yj2C9Nz;`hBfD11qQk_ zAm|ol_^j+VW&gA;*OL7fH^A!q4fX^T@ZW}MN5`Jyp&xu3=6XuX1Vdd9el!gNRWRawxENtt|(34A-=Vj&@9bX zW|7S!t#sSEtiK*TL%oV2Zddj5+SGRhb!$=b#GGBrd$!Z5GC|7=s;WoD)J>+&ul>MM z7@f|8X@+w$h)Ik5maAx@B*#kv`1iy+mR`Tp*)e;TmmO2lkAD8+I}tjF1D+x5OXy#n zJ)AJxyJe$|X*O~l;HGac$I^qYifEYZCaMQxC`5+*>?>SKz=)eo-Dc;9x*0d>|LpD; zRXWauH+~Tmx`c|Gz#bDCz5ZiwFqXJYEmJv_oktVdJ{D6!@Kwc+dq^vl>uLgCJtJ@? zcUP(CVG!hK2QgtswJair99wlYg&85Bxl`n6MjaS{Q2%(Va+WF!Y;P#in%%V=iy0EN z@l`qH!mN2BF@=W3Z;z7kMw-0)HCVCA84bMGYhT5WQsC)eLer0Lu!mAf(zK6WoIE~1 z>r;|GpB^;n(coOQP=f}TK79eP))9R-P%9b~`DW%{n^y0}pWlt|fAQ&EwQy*mQka~t zWy;D-X>_q(fj2F(b7V*{n;O2x*I84AqOJNoAm;l)%qU88A6;^0tMO%YYY%;pGrd)9 z^e52y&&n+3Lx!?G7(pTQK{BSTxr-?LTh1?F?J!2hy3i937dKaEN)4zzY95U3aWaVC z;}TSjRFC3yvIm%yA3cEQ`Wk2+)0~r08p@Gg^8@_abrceuM_%$=a4lHE=l?AQPvgC( zh5m)$TyQSDgtwoiU@=7CihqmYEtD#bJsZ(UNqt^fc4 literal 0 HcmV?d00001 diff --git a/contrib/site-packages/Pyro4/threadutil.py b/contrib/site-packages/Pyro4/threadutil.py new file mode 100644 index 00000000..524b3524 --- /dev/null +++ b/contrib/site-packages/Pyro4/threadutil.py @@ -0,0 +1,19 @@ +""" +Threading abstraction which allows for :mod:`threading2` use with a +transparent fallback to :mod:`threading` when it is not available. +Pyro doesn't use :mod:`threading` directly: it imports all +thread related items via this module instead. Code using Pyro can do +the same (but it is not required). + +Pyro - Python Remote Objects. Copyright by Irmen de Jong (irmen@razorvine.net). +""" + +from Pyro4 import config + +if config.THREADING2: + try: + from threading2 import * + except ImportError: + from threading import * +else: + from threading import * diff --git a/contrib/site-packages/Pyro4/threadutil.pyo b/contrib/site-packages/Pyro4/threadutil.pyo new file mode 100644 index 0000000000000000000000000000000000000000..71e6c5fd44b9fe929fcb24c14bef4bdab2eef480 GIT binary patch literal 801 zcmZ8f|BBN<5T2yH(~DfuH<Eh2 zNALl3HnARDAYU>&KfZ5fw)guu`u6MRG>3oB0ew&DwOUiA+kpv$)g-N0lx{3%3s*E^uQx?u4VLh|-j;nc>Rel9Do(uhHqd zNee2=6$*#KqS6l8mI=u+$&=_~V>A|;TlLtrrSA|fun>mlPBx>qKviqwtXCLyqR?>3 z952YrD~nsf(3QkPx;K)eP}WhO3BJ+=Cj#Ga%b7Dpyi|mvWfjLb+mOEh5)HrE5OMJ& zi8_rZl;%nr(Ue!(ar}^7lPsGcRjwN&R;9zN!Lg~RRie7k#1so}o*MRDo2^hhQQT2Q z@rRz6wgB#sAlGUsR-Nq+pHAX?-sfRHFLXYiJHI%u``MI!z(oYJchmE;*W=0CVQb#* z$UPwMxSiCwF*L6upR!+JOa|P7U6YyW+NDzGOs4NeW|(QxH-@IN(`IecuJL5KKS@7x zW9hcobeKF(rO48{9gVjvt>L7md%#xQXOiNyquw|n6AHXfd@s7~y!68CfZ#BE7#;<^ KU=SP!kHWu>I_XRR literal 0 HcmV?d00001 diff --git a/contrib/site-packages/Pyro4/tpjobqueue.py b/contrib/site-packages/Pyro4/tpjobqueue.py new file mode 100644 index 00000000..02de2be3 --- /dev/null +++ b/contrib/site-packages/Pyro4/tpjobqueue.py @@ -0,0 +1,203 @@ +""" +Thread pooled job queue that can grow and shrink its pool of worker threads. + +Pyro - Python Remote Objects. Copyright by Irmen de Jong (irmen@razorvine.net). +""" + +from __future__ import with_statement +import logging +import weakref +import time +import Pyro4.threadutil +try: + import queue +except ImportError: + import Queue as queue + + +log=logging.getLogger("Pyro4.tpjobqueue") + + +class NoJobAvailableError(queue.Empty): + pass + +class JobQueueError(Exception): + pass + + +class Worker(Pyro4.threadutil.Thread): + """ + Worker thread that picks jobs from the job queue and executes them. + If it encounters None as a job, it will stop running, regardless of the pool size. + If it encounters a lack of jobs for a short period, it will + attempt to stop running as well in an effort to shrink the thread pool. + """ + def __init__(self, pool): + super(Worker, self).__init__() + self.daemon = True + self.pool = weakref.ref(pool) + self.name = "Pyro-Worker-%d " % id(self) + self.job = None # the active job + + def run(self): + while True: + pool = self.pool() + if not pool: + break # pool's gone, better exit + try: + self.job = pool.getJob() + except NoJobAvailableError: + # attempt to halt the worker, if the pool size permits this + if pool.attemptHalt(self): + break + else: + continue + if self.job is None: + # halt the worker, regardless of the pool size + pool.halted(self) + break + else: + pool.setBusy(self) + try: + self.job() + pool.setIdle(self) + except: + pool.halted(self, True) + raise + + + +class ThreadPooledJobQueue(object): + """ + A job queue that is serviced by a pool of worker threads that grows or + shrings as demanded by the work load, between limits set by the + THREADPOOL_MINTHREADS and THREADPOOL_MAXTHREADS config items. + """ + def __init__(self): + self.lock = Pyro4.threadutil.Lock() + self.idle = set() + self.busy = set() + self.jobs = queue.Queue() + self.closed = False + for _ in range(Pyro4.config.THREADPOOL_MINTHREADS): + self.__spawnIdle() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def close(self): + """Close down the thread pool, signaling to all remaining worker threads to shut down.""" + count = self.workercountSafe + for _ in range(count): + self.jobs.put(None) # None as a job means: terminate the worker + log.debug("closing down, %d halt-jobs issued", count) + self.closed = True + + def drain(self): + """Wait till the job queue has been emptied.""" + if not self.closed: + raise JobQueueError("can't drain a job queue that hasn't been closed yet") + while not self.jobs.empty() and self.workercountSafe: + # note: this loop may never end if a worker's job remains busy or blocked, + # we simply assume all workers eventually terminate their jobs... + time.sleep(0.1) + time.sleep(0.05) + if self.workercountSafe > 0: + raise JobQueueError("there are still active workers") + + @property + def workercount(self): + return len(self.idle) + len(self.busy) + + @property + def workercountSafe(self): + with self.lock: + return len(self.idle) + len(self.busy) + + @property + def jobcount(self): + return self.jobs.qsize() + + def __repr__(self): + return "<%s.%s at 0x%x, %d idle, %d busy, %d jobs>" % \ + (self.__class__.__module__, self.__class__.__name__, id(self), len(self.idle), len(self.busy), self.jobcount) + + def process(self, job): + """ + Add the job to the general job queue. Job is any callable object. + If there's no idle worker available to service it, a new one is spawned + as long as the pool size permits it. + """ + with self.lock: + if self.closed: + raise JobQueueError("job queue is closed") + self.jobs.put(job) + if self.jobcount > 0: + if not self.idle: + self.__spawnIdle() + spawnamount = self.jobcount + while spawnamount > 1: + self.__spawnIdle() + spawnamount -= 1 + + def setIdle(self, worker): + with self.lock: + self.busy.remove(worker) + self.idle.add(worker) + + def setBusy(self, worker): + with self.lock: + self.idle.remove(worker) + self.busy.add(worker) + + def halted(self, worker, crashed=False): + """Called by a worker when it halts (exits). This removes the worker from the bookkeeping.""" + with self.lock: + self.__halted(worker) + + def __halted(self, worker): + # Lock-free version that is used internally + if worker in self.idle: + self.idle.remove(worker) + if worker in self.busy: + self.busy.remove(worker) + log.debug("worker halted: %s", worker.name) + + def attemptHalt(self, worker): + """ + Called by a worker to signal it intends to halt. + Returns true or false depending on whether the worker was actually allowed to halt. + """ + with self.lock: + if self.workercount > Pyro4.config.THREADPOOL_MINTHREADS: + self.__halted(worker) + return True + return False + + def getJob(self): + """ + Called by a worker to obtain a new job from the queue. + If there's no job available in the timeout period given by the + THREADPOOL_IDLETIMEOUT config item, NoJobAvailableError is raised. + """ + if self.closed: + return None + try: + return self.jobs.get(timeout=Pyro4.config.THREADPOOL_IDLETIMEOUT) + except queue.Empty: + raise NoJobAvailableError("queue is empty") + + def __spawnIdle(self): + """ + Spawn a new idle worker if there is still room in the pool. + (must only be called with self.lock acquired) + """ + if self.workercount >= Pyro4.config.THREADPOOL_MAXTHREADS: + return + worker = Worker(self) + self.idle.add(worker) + log.debug("spawned new idle worker: %s", worker.name) + worker.start() diff --git a/contrib/site-packages/Pyro4/tpjobqueue.pyo b/contrib/site-packages/Pyro4/tpjobqueue.pyo new file mode 100644 index 0000000000000000000000000000000000000000..4df08178edcd3c461e242048d5106afd8fcd3250 GIT binary patch literal 10078 zcmd5?%X1t@8SmMbv}?)oJ27!c=OMff_R1uw3WSj0S1JyU?2MeqD8Sb2PHQ#NXl6Y< zYe^}m$`uC=TvA0*ML|&%Ip)rVDlSl5xN+jZl^a+70QmjBo_#2`i{gMK+0*-VPft&G zf3M&7^_2fL+xXExKKHw-_^aagyZ4l`{{#^#wSv-9VM)cF+AOJ+lJZzDtGKLI%5o2M zMXgkZbXBcXhjdM?OsSni!|JqJnO4im)2d?XYNMgDf%3fJcn#i6Jax;I)%pPyR@8k@ zt<0#fs?4-npH-`PZ%uuw)P1GCzj9E8Q_9Rq>-4AI^RxUrL^`kcP8ldq+T_RVIR8-2aAZo0W`X?-c}?_^PL zE!UkLeKp%Oi4Ki^Crx^~8L|4^EchtR?nH@cB_==9YDE9WU(>@(^Mfe1qI|7w^B_0q zoG;o#XyZFyY2|&)U8bHNMdM=nPTIL}Cy3&p6PqhpmS!WSbF_>1umRH`44CdIf2j#t zHA^URHn-bJuxZ-uydjm%G#tdJx?X5fLfH*a3u!-Jh|_KmFIOvfK7W%STtl20x=lVf+Bj_1EiFGZE*VXFp_?-H2T$U59 zc{%xrg}z6Y{ZEVZIe=eAvDZtgU=B~>8ZM~@dRgS^L56AEa}1-dmZ)e@5GrO#hzc3W zMcMP%sfNa%8>$ddC)NF^yJ0EnRuiuDQPMlBGt&#QFgDgwUHBp@nzqqL<`KgNI)(tS zx0_&^;SF{z&2rs0S(JvOSIM-39IDXIb)N2eGbgrXFr6rY?rO8T$^qEPDKTf_v{okM z=F+81@Zt_$x`$m{1d-#vr|!ZImDGBP*2v4tthXyFud4N$y6e$)v1N)Jf&Q?!r%te` zPP0k^Hcr9vaQsC(hm!Jr&avm*e5jiIY09blzCsX`T_B+8+R6|~&U1}3M7gaM38 zl23u$GQmb>RxvB3_{)CW=A7?}La9@mg&^CcC$`tAJG)yGulk(8SQ{R@ZgVNmb1pPF z$~)@Sy(1{Q)OoIi8zNl!3yAGMkulkyxwY%qf2BSK3p{0?Q8yB$c8;>**u~8+!w;QS z73={0(1-Lxyk9Lim6g?v+dojZ%i@tZBveYNjcJwLm4T*;j@}LQ;gdridpl3b6v`qP zyi*@`T}Id9J@Po_3lZ`@QGCxWT+4XHlGygeCfDpwvk);LhkGV}C++w&@nZM#1E30d ze>;eCpF$+Eu!fomMHZ~dFAnU^kiH7t6E6FQIn3U2*a_$$c@a~wB4}Xoj92q2UKxy@ zEzNnA(v&w>n#J!d+8W+rl!v?~SwFVevbc6TOuOy2PpR>rKrxnFp&q;Va*n5PS@Nz{ zt9kXQ{1`|FFm4FG06)c5bpe?uW=`ND)l5hN8^Z4H1;7a$ARCwK&ef>8$SBIJU@ z$Ydz7*gg0e3#_yW@8m|H1t-GLaT>rUbWFZw0H@+;lK>5Bz}KLYjB)dA|H_5SH?Cj5 z)_(WuqN^)0H&jGdlg34!PoLRMgkQ}@G!$V@rF!Y=lmD}&{vJ|uj;wA(i2|jb4 zY>-DWUa8czw7Ve$K-q>U9kxRnBA5dqI4}u-sD5FDnTP=yW7rp`bLZ}D|g_0C=GI>vxh66#w4)vZDOTcpoa_R za>~Ueta}#~w{TzvDxe?-QN0uDDXI@~ObEv?a&@ zhqp^8E>TZ(m~JHkb|)a_S%BVN62yST#GwJOX$Er!Xom(GvT+>gaOxnJaa#6Sj7v_i zBS$`~ffniV&rwlzWbMF&i%l{`b z9Q>F^v#iIloEk4f`SjX*$6fy{UjHF3hYm}10ET0L24a_o92V}vCKuebPm;CZ8JPHq zrvMALUmP>{zEU&UWvN#bh=RjF_+UdlvDwdiwANCB2 z{n61?Z3uiU+sX%qs6o=XQ@h!F_WqDvT&w=+`?TPI9wNjnlpP8oZCz}>O*$NL(O6qSlqjvB$=QlDiXkfK8r#`(O0Cw^iDABh#{2bo+nNWC z&h`22`EBtc#DhW*4+=q-@R!0Ou7vAK!zGSEha=90b;O{ID((b05xTo zi#@*&Bn3aihpL1U+o#oCj8~T4Kbh#{TEB@dx4Xzqz-hp@KrbuMOEJ5D#q1>VVDxbz z3`f2M;DNDE&m<-b;<3YMA&=Yv8bDaHgT!?#iC>*c?%BmK90_SW@err2PEt+&55Y$; zOco3LaIqR9ptCyAiP_T3_A!VfZa3j*W^idS)7SXrtVY>AChH^;i}LZy?NKah><=-V z^J(I}B5L%ah@LOzXdd;klk#6;m2&1x6dmmtr%0AW5+pLgV3S$Oy+bPkj{8~KMdI@w zCTdThP_+hPi8+84#uT$c!=TYl@fykHE2tE5IX{$3Sn>(^o0P`~>^Y^!ZJ-7v&>TQB zy<=qG#M1>~I8F*l59ej}Zo__V zk39@!JpBt?lUqT<#cL!a&!SQY$&sOu6cVxw^94)d!IE$XVVhv^U=N0wmm=8MS_2XV zP-P@xb(3+EJ=4-R*Pxb8V4O-9-A5+{oiyFp0478@(pq%8Ms>|uJR;b4`N;`wS!h9oSp9-5WJJ2}jc@JFh^S?|Q&7pofbq5JXP) zfbKDdkS4eXhLIt07+<((nV>E9+U?hCdfqn4aA)ww;`%b%XeIoEDE5uu7KasX@}Fa7 zMW{B1P>t7v`uqku6+*l)6k@nv=WiKkotN9l(z{)Tsak^A0K@K<@ZVsZkwTt8wa}38G27~bwf3R=R=G& zfPx{d8OP}s^z4DLC;i&DF@pNUimFhG<*Aga)#VuI>0wTX zMsZ|;KS0Gv7lK(}Ay|lws>)8%m@c`<2a>}RKmp#PGg+qTkamY)!GhpqjwY8>17av& zwiKy5)W#yVM2M>$>YW70Q30I=21{-wOL2N$hz*hNn5)grfz4qtU=un<&^W9DKi4?g z5Qj1&yCoaP6qk!5-XwK_g$zmDCFa*BCU7R1_D=-Wmv{rUy+np5@%p!LSvrYY6-SKq zdKv$w>oaqex$0b{Ua23cSL@Z8D%uXzXWd?Lhl=lbRtNBL_>RX%lS5!g0r%kD{018Q zSJ}z~rsA6-`L2oQO3u=VC7}pj-R$Fg3i;s1e~!=8QNSQw!(jMyK?Lo4XmaP)ym@82 zYmoFz6Bp4@&L!}0r$g>z2Vxx;ulii+7!(TixN*3M&g<^Or#I;5>?str(jhq(;?WR~ Srf0oF<+<|7@`=i+7ykpHGHaRu literal 0 HcmV?d00001 diff --git a/contrib/site-packages/Pyro4/util.py b/contrib/site-packages/Pyro4/util.py new file mode 100644 index 00000000..9c365d1f --- /dev/null +++ b/contrib/site-packages/Pyro4/util.py @@ -0,0 +1,546 @@ +""" +Miscellaneous utilities. + +Pyro - Python Remote Objects. Copyright by Irmen de Jong (irmen@razorvine.net). +""" + +import sys, zlib, logging +import traceback, linecache +try: + import copyreg +except ImportError: + import copy_reg as copyreg +import Pyro4 +import Pyro4.errors +import Pyro4.message + +log=logging.getLogger("Pyro4.util") + + +def getPyroTraceback(ex_type=None, ex_value=None, ex_tb=None): + """Returns a list of strings that form the traceback information of a + Pyro exception. Any remote Pyro exception information is included. + Traceback information is automatically obtained via ``sys.exc_info()`` if + you do not supply the objects yourself.""" + def formatRemoteTraceback(remote_tb_lines): + result=[" +--- This exception occured remotely (Pyro) - Remote traceback:"] + for line in remote_tb_lines: + if line.endswith("\n"): + line = line[:-1] + lines = line.split("\n") + for line in lines: + result.append("\n | ") + result.append(line) + result.append("\n +--- End of remote traceback\n") + return result + try: + if ex_type is not None and ex_value is None and ex_tb is None: + # possible old (3.x) call syntax where caller is only providing exception object + if type(ex_type) is not type: + raise TypeError("invalid argument: ex_type should be an exception type, or just supply no arguments at all") + if ex_type is None and ex_tb is None: + ex_type, ex_value, ex_tb=sys.exc_info() + + remote_tb=getattr(ex_value, "_pyroTraceback", None) + local_tb=formatTraceback(ex_type, ex_value, ex_tb, Pyro4.config.DETAILED_TRACEBACK) + if remote_tb: + remote_tb=formatRemoteTraceback(remote_tb) + return local_tb + remote_tb + else: + # hmm. no remote tb info, return just the local tb. + return local_tb + finally: + # clean up cycle to traceback, to allow proper GC + del ex_type, ex_value, ex_tb + + +def formatTraceback(ex_type=None, ex_value=None, ex_tb=None, detailed=False): + """Formats an exception traceback. If you ask for detailed formatting, + the result will contain info on the variables in each stack frame. + You don't have to provide the exception info objects, if you omit them, + this function will obtain them itself using ``sys.exc_info()``.""" + if ex_type is not None and ex_value is None and ex_tb is None: + # possible old (3.x) call syntax where caller is only providing exception object + if type(ex_type) is not type: + raise TypeError("invalid argument: ex_type should be an exception type, or just supply no arguments at all") + if ex_type is None and ex_tb is None: + ex_type, ex_value, ex_tb=sys.exc_info() + if detailed and sys.platform!="cli": # detailed tracebacks don't work in ironpython (most of the local vars are omitted) + def makeStrValue(value): + try: + return repr(value) + except: + try: + return str(value) + except: + return "" + try: + result=["-"*52+"\n"] + result.append(" EXCEPTION %s: %s\n" % (ex_type, ex_value)) + result.append(" Extended stacktrace follows (most recent call last)\n") + skipLocals=True # don't print the locals of the very first stackframe + while ex_tb: + frame=ex_tb.tb_frame + sourceFileName=frame.f_code.co_filename + if "self" in frame.f_locals: + location="%s.%s" % (frame.f_locals["self"].__class__.__name__, frame.f_code.co_name) + else: + location=frame.f_code.co_name + result.append("-"*52+"\n") + result.append("File \"%s\", line %d, in %s\n" % (sourceFileName, ex_tb.tb_lineno, location)) + result.append("Source code:\n") + result.append(" "+linecache.getline(sourceFileName, ex_tb.tb_lineno).strip()+"\n") + if not skipLocals: + names=set() + names.update(getattr(frame.f_code, "co_varnames", ())) + names.update(getattr(frame.f_code, "co_names", ())) + names.update(getattr(frame.f_code, "co_cellvars", ())) + names.update(getattr(frame.f_code, "co_freevars", ())) + result.append("Local values:\n") + for name in sorted(names): + if name in frame.f_locals: + value=frame.f_locals[name] + result.append(" %s = %s\n" % (name, makeStrValue(value))) + if name=="self": + # print the local variables of the class instance + for name, value in vars(value).items(): + result.append(" self.%s = %s\n" % (name, makeStrValue(value))) + skipLocals=False + ex_tb=ex_tb.tb_next + result.append("-"*52+"\n") + result.append(" EXCEPTION %s: %s\n" % (ex_type, ex_value)) + result.append("-"*52+"\n") + return result + except Exception: + return ["-"*52+"\nError building extended traceback!!! :\n", + "".join(traceback.format_exception(*sys.exc_info())) + '-'*52 + '\n', + "Original Exception follows:\n", + "".join(traceback.format_exception(ex_type, ex_value, ex_tb))] + else: + # default traceback format. + return traceback.format_exception(ex_type, ex_value, ex_tb) + + +all_exceptions = {} +if sys.version_info < (3, 0): + import exceptions + for name, t in vars(exceptions).items(): + if type(t) is type and issubclass(t, BaseException): + all_exceptions[name] = t +else: + import builtins + for name, t in vars(builtins).items(): + if type(t) is type and issubclass(t, BaseException): + all_exceptions[name] = t +for name, t in vars(Pyro4.errors).items(): + if type(t) is type and issubclass(t, Pyro4.errors.PyroError): + all_exceptions[name] = t + + +class SerializerBase(object): + """Base class for (de)serializer implementations (which must be thread safe)""" + __custom_class_to_dict_registry = {} + + def serializeData(self, data, compress=False): + """Serialize the given data object, try to compress if told so. + Returns a tuple of the serialized data (bytes) and a bool indicating if it is compressed or not.""" + data=self.dumps(data) + return self.__compressdata(data, compress) + + def deserializeData(self, data, compressed=False): + """Deserializes the given data (bytes). Set compressed to True to decompress the data first.""" + if compressed: + data=zlib.decompress(data) + return self.loads(data) + + def serializeCall(self, obj, method, vargs, kwargs, compress=False): + """Serialize the given method call parameters, try to compress if told so. + Returns a tuple of the serialized data and a bool indicating if it is compressed or not.""" + data=self.dumpsCall(obj, method, vargs, kwargs) + return self.__compressdata(data, compress) + + def deserializeCall(self, data, compressed=False): + """Deserializes the given call data back to (object, method, vargs, kwargs) tuple. + Set compressed to True to decompress the data first.""" + if compressed: + data=zlib.decompress(data) + return self.loadsCall(data) + + def loads(self, data): + raise NotImplementedError("implement in subclass") + + def loadsCall(self, data): + raise NotImplementedError("implement in subclass") + + def dumps(self, data): + raise NotImplementedError("implement in subclass") + + def dumpsCall(self, obj, method, vargs, kwargs): + raise NotImplementedError("implement in subclass") + + def __compressdata(self, data, compress): + if not compress or len(data)<200: + return data, False # don't waste time compressing small messages + compressed=zlib.compress(data) + if len(compressed)") + if "__" in classname: + raise Pyro4.errors.SecurityError("refused to deserialize types with double underscores in their name: "+classname) + if classname.startswith("Pyro4.core."): + if classname=="Pyro4.core.URI": + uri = Pyro4.core.URI.__new__(Pyro4.core.URI) + uri.__setstate__(data["state"]) + return uri + elif classname=="Pyro4.core.Proxy": + proxy = Pyro4.core.Proxy.__new__(Pyro4.core.Proxy) + state = data["state"] + uri = Pyro4.core.URI(state[0]) + oneway = set(state[1]) + timeout = state[2] + proxy.__setstate__((uri, oneway, timeout)) + return proxy + elif classname=="Pyro4.core.Daemon": + return Pyro4.core.Daemon.__new__(Pyro4.core.Daemon) + elif classname.startswith("Pyro4.util."): + if classname=="Pyro4.util.PickleSerializer": + return PickleSerializer() + elif classname=="Pyro4.util.MarshalSerializer": + return MarshalSerializer() + elif classname=="Pyro4.util.JsonSerializer": + return JsonSerializer() + elif classname=="Pyro4.util.SerpentSerializer": + return SerpentSerializer() + elif classname.startswith("Pyro4.errors."): + errortype = getattr(Pyro4.errors, classname.split('.', 2)[2]) + if issubclass(errortype, Pyro4.errors.PyroError): + return SerializerBase.make_exception(errortype, data) + elif classname == "Pyro4.futures._ExceptionWrapper": + ex = SerializerBase.dict_to_class(data["exception"]) + return Pyro4.futures._ExceptionWrapper(ex) + elif classname.startswith("builtins."): + exceptiontype = getattr(builtins, classname.split('.', 1)[1]) + if issubclass(exceptiontype, BaseException): + return SerializerBase.make_exception(exceptiontype, data) + elif classname.startswith("exceptions."): + exceptiontype = getattr(exceptions, classname.split('.', 1)[1]) + if issubclass(exceptiontype, BaseException): + return SerializerBase.make_exception(exceptiontype, data) + elif classname in all_exceptions: + return SerializerBase.make_exception(all_exceptions[classname], data) + # try one of the serializer classes + for serializer in _serializers.values(): + if classname == serializer.__class__.__name__: + return serializer + raise Pyro4.errors.ProtocolError("unsupported serialized class: "+classname) + + @staticmethod + def make_exception(exceptiontype, data): + ex = exceptiontype(*data["args"]) + if "attributes" in data: + # restore custom attributes on the exception object + for attr, value in data["attributes"].items(): + setattr(ex, attr, value) + return ex + + def recreate_classes(self, literal): + t = type(literal) + if t is set: + return set([self.recreate_classes(x) for x in literal]) + if t is list: + return [self.recreate_classes(x) for x in literal] + if t is tuple: + return tuple(self.recreate_classes(x) for x in literal) + if t is dict: + if "__class__" in literal: + return self.dict_to_class(literal) + result = {} + for key, value in literal.items(): + result[key] = self.recreate_classes(value) + return result + return literal + + def __eq__(self, other): + """this equality method is only to support the unit tests of this class""" + return isinstance(other, SerializerBase) and vars(self)==vars(other) + + def __ne__(self, other): + return not self.__eq__(other) + __hash__=object.__hash__ + + +class PickleSerializer(SerializerBase): + """ + A (de)serializer that wraps the Pickle serialization protocol. + It can optionally compress the serialized data, and is thread safe. + """ + serializer_id = Pyro4.message.SERIALIZER_PICKLE + + def dumpsCall(self, obj, method, vargs, kwargs): + return pickle.dumps((obj, method, vargs, kwargs), pickle.HIGHEST_PROTOCOL) + + def dumps(self, data): + return pickle.dumps(data, pickle.HIGHEST_PROTOCOL) + + def loadsCall(self, data): + return pickle.loads(data) + + def loads(self, data): + return pickle.loads(data) + + @classmethod + def register_type_replacement(cls, object_type, replacement_function): + def copyreg_function(obj): + return replacement_function(obj).__reduce__() + try: + copyreg.pickle(object_type, copyreg_function) + except TypeError: + pass + + +class MarshalSerializer(SerializerBase): + """(de)serializer that wraps the marshal serialization protocol.""" + serializer_id = Pyro4.message.SERIALIZER_MARSHAL + + def dumpsCall(self, obj, method, vargs, kwargs): + return marshal.dumps((obj, method, vargs, kwargs)) + + def dumps(self, data): + try: + return marshal.dumps(data) + except (ValueError, TypeError): + return marshal.dumps(self.class_to_dict(data)) + + def loadsCall(self, data): + obj, method, vargs, kwargs = marshal.loads(data) + vargs = self.recreate_classes(vargs) + kwargs = self.recreate_classes(kwargs) + return obj, method, vargs, kwargs + + def loads(self, data): + return self.recreate_classes(marshal.loads(data)) + + @classmethod + def register_type_replacement(cls, object_type, replacement_function): + pass # marshal serializer doesn't support per-type hooks + + +class SerpentSerializer(SerializerBase): + """(de)serializer that wraps the serpent serialization protocol.""" + serializer_id = Pyro4.message.SERIALIZER_SERPENT + + def dumpsCall(self, obj, method, vargs, kwargs): + return serpent.dumps((obj, method, vargs, kwargs)) + + def dumps(self, data): + return serpent.dumps(data) + + def loadsCall(self, data): + obj, method, vargs, kwargs = serpent.loads(data) + vargs = self.recreate_classes(vargs) + kwargs = self.recreate_classes(kwargs) + return obj, method, vargs, kwargs + + def loads(self, data): + return self.recreate_classes(serpent.loads(data)) + + @classmethod + def register_type_replacement(cls, object_type, replacement_function): + def custom_serializer(object, serpent_serializer, outputstream, indentlevel): + replaced = replacement_function(object) + if replaced is object: + serpent_serializer.ser_default_class(replaced, outputstream, indentlevel) + else: + serpent_serializer._serialize(replaced, outputstream, indentlevel) + serpent.register_class(object_type, custom_serializer) + + +class JsonSerializer(SerializerBase): + """(de)serializer that wraps the json serialization protocol.""" + serializer_id = Pyro4.message.SERIALIZER_JSON + + __type_replacements = {} + def dumpsCall(self, obj, method, vargs, kwargs): + data = {"object": obj, "method": method, "params": vargs, "kwargs": kwargs} + data = json.dumps(data, ensure_ascii=False, default=self.default) + return data.encode("utf-8") + def dumps(self, data): + data = json.dumps(data, ensure_ascii=False, default=self.default) + return data.encode("utf-8") + def loadsCall(self, data): + data=data.decode("utf-8") + data = json.loads(data) + vargs = self.recreate_classes(data["params"]) + kwargs = self.recreate_classes(data["kwargs"]) + return data["object"], data["method"], vargs, kwargs + def loads(self, data): + data=data.decode("utf-8") + return self.recreate_classes(json.loads(data)) + def default(self, obj): + replacer = self.__type_replacements.get(type(obj), None) + if replacer: + obj = replacer(obj) + return self.class_to_dict(obj) + @classmethod + def register_type_replacement(cls, object_type, replacement_function): + cls.__type_replacements[object_type] = replacement_function + + +"""The various serializers that are supported""" +_serializers = {} +_serializers_by_id = {} +def get_serializer(name): + try: + return _serializers[name] + except KeyError: + raise Pyro4.errors.ProtocolError("serializer '%s' is unknown or not available" % name) + +def get_serializer_by_id(sid): + try: + return _serializers_by_id[sid] + except KeyError: + raise Pyro4.errors.ProtocolError("no serializer available for id %d" % sid) + +# determine the serializers that are supported +try: + import cPickle as pickle +except ImportError: + import pickle +assert pickle.HIGHEST_PROTOCOL>=2, "pickle needs to support protocol 2 or higher" +_ser = PickleSerializer() +_serializers["pickle"] = _ser +_serializers_by_id[_ser.serializer_id] = _ser +import marshal +_ser = MarshalSerializer() +_serializers["marshal"] = _ser +_serializers_by_id[_ser.serializer_id] = _ser +try: + import json + _ser = JsonSerializer() + _serializers["json"] = _ser + _serializers_by_id[_ser.serializer_id] = _ser +except ImportError: + pass +try: + import serpent + if '-' in serpent.__version__: + ver = serpent.__version__.split('-', 1)[0] + else: + ver = serpent.__version__ + ver = tuple(map(int, ver.split("."))) + if ver<(1,3): + raise RuntimeError("requires serpent 1.3 or better") + _ser = SerpentSerializer() + _serializers["serpent"] = _ser + _serializers_by_id[_ser.serializer_id] = _ser +except ImportError: + #warnings.warn("serpent serializer not available", RuntimeWarning) + pass +del _ser + + +def resolveDottedAttribute(obj, attr, allowDotted): + """ + Resolves a dotted attribute name to an object. Raises + an AttributeError if any attribute in the chain starts with a '``_``'. + If the optional allowDotted argument is false, dots are not + supported and this function operates similar to ``getattr(obj, attr)``. + """ + if allowDotted: + attrs = attr.split('.') + for i in attrs: + if i.startswith('_'): + raise AttributeError('attempt to access private attribute "%s"' % i) + else: + obj = getattr(obj, i) + return obj + else: + return getattr(obj, attr) + + +def excepthook(ex_type, ex_value, ex_tb): + """An exception hook you can use for ``sys.excepthook``, to automatically print remote Pyro tracebacks""" + traceback = "".join(getPyroTraceback(ex_type, ex_value, ex_tb)) + sys.stderr.write(traceback) + + +def fixIronPythonExceptionForPickle(exceptionObject, addAttributes): + """ + Function to hack around a bug in IronPython where it doesn't pickle + exception attributes. We piggyback them into the exception's args. + Bug report is at http://ironpython.codeplex.com/workitem/30805 + """ + if hasattr(exceptionObject, "args"): + if addAttributes: + # piggyback the attributes on the exception args instead. + ironpythonArgs = vars(exceptionObject) + ironpythonArgs["__ironpythonargs__"] = True + exceptionObject.args += (ironpythonArgs,) + else: + # check if there is a piggybacked object in the args + # if there is, extract the exception attributes from it. + if len(exceptionObject.args) > 0: + piggyback = exceptionObject.args[-1] + if type(piggyback) is dict and piggyback.get("__ironpythonargs__"): + del piggyback["__ironpythonargs__"] + exceptionObject.args = exceptionObject.args[:-1] + exceptionObject.__dict__.update(piggyback) diff --git a/contrib/site-packages/Pyro4/util.pyo b/contrib/site-packages/Pyro4/util.pyo new file mode 100644 index 0000000000000000000000000000000000000000..7272c1a545507ce0c835b06b14f0a4fa600fc3da GIT binary patch literal 26310 zcmds@l``()&CA;lrS5!mk-MR1H`|gi(&pqedbI!e`KixU-h5z)8u%GyQ%){!-1~&4#lIATNre;lDkEhcB*gb{~mXn&g{-! zi!P$|ZSFR`+Ux3!WuJ=*ZehQxJ>nLfYoE`WKj5YgxV7JQ(Lom-P_+w(+%4xWTs!OD zXQ+qW)B`RWa?u0IBeK}%+O7~isFdi4yH&JT;xusBonvJDpKM;shH%tP9d&EtR{zoe z%lfA`*3WCF{^`QXbYXIB(kkqC3lDYI`RXV4`Qxlk`)1)`>p6*?nm;bAd|a5U^||N~ zH_Kf2yW38n`21AhY6C6`EOkeodd$@Zv(%k=>Z7i^wi7xDkNs~vFX6njGN3#Z(< z6vIxtsnafcbYt0ro64>~oz38gn;LP^;~R?~+Eje9xi`n|(Gq=Af7X9BtK*EDI^$}G zU3A7RJna^HUHgkAA76Mz={3*`T;A@6rk-_8@Z8D~kDhbUXI%8Wi=K5VI;82QsPLcM zXI(8YeEwc3dcoBmRQg8+*K{s=Y-5Lyd`gG^d)DFio^rLL2K*OY?IDZ2!;b9b8#Y=3C7mjFwtS6kMKJ zh^k3E5(J~I<@I)bZaxWS)`N@frKlOyqTsDob1oRJ>-E)k<%3rHdc7HqG^6C?$Uyyn z@i!h1(5&kI%!pd6mU!*u>EjvYiT@)3a|{;UEV&Q<(0#~jk6Y|_?W^uXFq$~4mpGu# z;z>cp02>isGBKj!0l9KjuuSXW+uL2=-gXJps7GLazvOPxXu;hqs(iA|nu9cwOID9d zdQ}O1py_10Tj+BO{q9zeyGh9&*Z!gVu%rl$QusqlJUqeg#3WUG(YY@c+!u?G$R1`u zWI8StoC~95wcU(^O3e=wg>FJ<8YqhPnR)bnAXtt6dUR_@1Lye^6aVQ|S z9Y>AXkt$$tJ?S$czTjcP2g)NCo|;+^AlT{b`-BRl0;>cXf;^w zFe%Q0I3jkYwVa%3w3y*Dm+CX^N_+jx>+Q-?bfeW?jL&#FFfyB^ocUbTjzP8f%(Eje zo@vx)&MX_H&|6$jqSMRZYGp3cmn`-(TI?efIY>($e?63Pj2j{&K0x3K{e_){-RZNl zaFAI1dD836>y<{m7F61Es}PvvER#`A)|aCoo^P!-YQantRGM8gq`Xr>s~s$?vb0hZ zY_{?WK$cfF4@ON7sjrhzj$fvLiuWVest`$^P_JY4qAoj zP}7#mKw{NTr7~thwP}6JF~CaNbdtX%Q@zK-K|{?4uL+Ki61Z*sJ4%Jei~9=u3J(+> z;P*(Ow=h`fEA^GCg7!|HL7q{b_}c&)A^?XKd4JO=GjWJHan^Mf-^>W!bYcWBTjQuK z^^_6#@@rk+u6-gSfQYnzxg<8RO@wFBUTYC?LQ2 zSJH=&pci&#rFU5yH~U?4u#?1V+L2EGy1KE$eb}e*FYI<7_Pg2AhCU8@D(E}B({sGVx;fw$4mj4j9}U8!WF8av3!C#0_mOz+o`?94 zbbj}|y^?aWJl|3J1VLZ>rb`Z~3%p#o_LtpFM#}#jkk1!=oW!AryT_rsZ(^C)YIi#0&oBPvhx-%zOg%iUMX}LZKzkA(;6iDm_ik!(8Bf-U4NdQ45 zUX+l7@Q6s!fSr13nIHn4G9f}DgJ)sEje4UIKrbbHm>d%z0xAD`rCqPgG$M&JK~$;E zBScA5GH&H1o6nmp(mas_^Ofs}Q>|dR-MU^!W>p0pt}YoS6H!hf$XF*^OZ7zMma;A( z2+XcFt0se4ue?NLWrBJlu_joJ8AEWVbTsl)W@eB|S8dd*k~#F~-K47LiLCE(iIGwk z$HZha&>miCnE(v}J(KS1nPzS1`FOaj~_jS#NUj9<2BH%X>Ez+O&}Fhd$m=I&JLKcDDghoK~}W|+iy1! z$-Kb9yMIYDwUMJhoE4sJN0Hub_?INuu}}a>OvM6|Dn1?uFI(G^%aou$lgjfPP2+%x zJHgCqy-^cZM;Sm27TT5^J9aEMI{EWTQsj%8nLgH7Mpgi0_F9rMev(ItpjUGY|@ZB7D|okASL2u~0ML znbLc=tq8UG`5ER+R4|%|;69D|2T}XAN*q;{u0J8^ zYdrCf2-sjr`$||eOe6zI zsK&lD@42BDG%M|)dT$DXP=GM%pd?%7S(&X^i-~AY4w4IW>1#^pA#kPLg?t{hK5QP3 z6G6UgKOoV>PoJO7rk`I_H8#KLY)TbyA<7$&qAUu!CNzcqNH7t>kW*j*V@C zvD8Ga*vmG%s;HQo5X>O2j^-~+3X^YYJf zMM|Os{m@TzRsEQ&Nj%$D((i!&Z+f#uD*D3A2=9p*?sFQ&a z#ty~&vDaYumO%bCftR|p>NNCCD=&P95?eAUo7S89_D%ZY*IUWfn`002nxtQ_GsAdwmeQc<_W!M>BqHDj-6Kklh zP|ZfH-#^=^!%3>RABOl@`th4Qe~yPZ#_;|@*AT@c^!%h|QFVDly`tF9D+m=xHL(>7 zIJ`{5|A;4>RPh2aem#b_UmyZg9E^udXJU(%rMrlk5GMu+H74kNFNNqhc%axwtj z!B7G+SwtQ{^20dV0oKcG8aBZz)I#=bL=XtedgbxKy;D*o576dn^X>!wm*jbouE-uq zL(V9$;lzJs`q!RGhk4=J6EYe|X`AhI+ZbIuz7*Y%Cp!}>18x%H9_dM^#;D9&qn!+%_6DdZb{Vy=0@yDx}W;W9#~CviVo2{*YDwUsQcMR7`YK{rv(u z9yt;`Zhbn8y$7oeKzu;&^7P1dOy^cUDwqSo4%x}>I#@N=0eAl=QL^+EufeU#Q?`(lq^p zwgRSE9|@nLD%%#z3|^mEMImAYr(BL3tpuHG{3s=AEp=tF*}4(T^DvUEjZA42pP|@D z@CFJSsZFFd9X=_233esCAKI;Dly7(@mCKmiQO_q)xja5B;%m#^x=_awLl#5ys^w?Z zH?dOz%{F|tYq2DG;e^Jdv|Em3<&RCHai!f#TGdv=W5cf5KAvA(!5|yZ$ z>0fOa0`iv7IPFz8Q1X98LVS+EmG&SN94tIsc!FQFu*E&Z_Y`{2sqR6$+5_QJd>^UO zw(dr0u?OAj!{k)O{WN|N#ZjJkUkM&y6c5@Ze<19zMx8)vqiJBg?rhiyW zYlZscO&cAbX#ScxJ5l#eeb~Nbquphny&-CMiyf$H+HFZw1rRq&+G9ye1yC_d+MBfp z!RN!~n_2B%grcIp%*M3qcOaKT_SnjyNkPp0^EZtxcX?DYSDTQFwYEcKS7p@&O zCkl8BeiVm?&aE~@^qa3BpNR0q7x-yMv#YUJ)OIN{19@@9*#47w zoC#OO_g1mRw&Q9GJ-sv~bg_>9oF1PIVi5^*t{PGPh}k~7-ai{&G=guOA{L$R%7nzP zt;6%t#E9P1)wfvEcvl(E?Yf#V$F^Afz?ZW3EA{GPBbvw!9maaQ^521t%~u*-1;nws z3cMA!nq9d^H|CNfj^0&3WY>T0S=LAxkzytipFF#YpDB*L<(wl;w&gw8Hm;l-4sCX1 z=EYPIkJvCfc_Z<|REivcGpAWHJKP%8uS}_DgYjYUb=#uBkd2du4W+1u6A>@n znXzW(p`CcoMloc~nSHBy5@*FI8{NPr>-i&ima=%8M4PYBxVe~hDu~#mZi0WJS-b=2 z#$-1_+6vwQUSqhSa3Yl_GUc2BPXP8DT3BqA$^)TejkU4cevI9X!O6y!PKh;n=d|=u zS2hu0D{Hzl8P*O;$4)b`hf>_ZI@4==o4*MwUf@`wvSF0=bv>ys;V|wE)P`_cq_&nZTWhBCep-giD4lL?gq z_3lFe88R{C2*%+a-j1NIIE28VQ13@AcRR7{ky7JP1epD#?Wg4K!XrE~82jJB!hzy` z#FxIi?BA}^D5M^0jkq%ORrVF|BioxvSh-?*JM!4PEx?034|pG5*+uRq3**vjeM7+JM_}56aDGy~B@&dc0RuPvinUh~s!>lSh z6W|aB2a69O@Kr^3wfKdD8ejaU1i3oR%VO|AxqrSQNdTD`z+Sj^Q1D`+<=kz8(iiZo zdsHlZ0$73C`eNbJi~qbY>rWPZO;DnjI!RM-zu=ppx~#@+R^yMlX7YEeedIP@NE^r= zw{}|Z$YwUNkC@NkHrd$HzV@m@1VCiGb1Of<6A&LeK1e6GD!L<(2o%rp7!?MYt{srb zt2(S>)bY8(3gWJe3i>A17Kt9hjZFyf)F+A!O>Di6JPiQRE!Y>PUNls=cUloI56ne4 zyezk0c}eJ?!S?#?)o>s2&aN%|n|fVTz@S|yW^GG56v}rQ_Jlq-$w%!-~b}yFjo8)#z6n;amJoj8x=m*n8N$`3;n8L7W9Zu1maG4+#kZP?ess!mcz1iTR zV&e+Lbi&UmxTHW3I zc0@o9TextM!s$y~wakl)RXR0@>C29H#9yOY+pe7tHn~UH5wIJ`>E6l3?<#Vk+kB?v z#%y{izcsrkA5=Dab_@*Za6R{cv4x!rVct{elx^`?-Cd4WzMiVqa`xD|u@%m`fm3yf zBdC(W5g0B+C|gJ+J~dQWHYwat&b$=gy!ghOV-r`)SHjCzFOOcngd_Iy-e zXEv28;Kw5haz6}3MC{!^A43sH-|HHRaIA+%%TDq5X9>EcPo&#*j@4w?CmFY>bLz5@ za4-oQ$qVw`-{`hqr3I%)b1CdKtp_CBOAGM9Tbl&N0Sc8d9oAN>@EofK1Gr-_wZYaY2Y?6eEwrjQ+oBpSa|XLrHfNzVfo6%(YG&+ z*^WCW?d@{C<~jVBsxYoER13eYKn%V^gYNM}8ee@|W0w@x+rxpsV#bHVk4UQW*q@N~ zHJ+aq&wZJ?Kb_ewc@^C!aoal?q%?S)wVCbLVUU~Xsh>|@yHV!*lnruW$Q^7@0^X>m zMRT*hw=-UiakBZO5h6BN_{E4Ey)1H+!UU>ib{I;kus>Kt6KqpO@WOwnKp1N(Hh*?U z);E6;gi7uU_M;BA1eiV|I#RS31(0nr1R7Cq1CM0z92Xew`?%C*+8iAaHIa#JK3V7o z(pNiruG|cF&rXFFl4NQ1Y>u^=xT*m_^%o$OK4&P(; z!%Y;Kcg-iVo$$BmwH-^J=Pq-*LwskxPo-I4hPl?B*wcpP?<`;8B>?pZ3Ew+k2byM;&rToZLzC#1IWKD@H zd}Y&6?hQXMF#LbP4}MAT@m%vKzz_0q-!J^&>s`|?ULY2rUp?_-1e-Q;xpa{3;*4Dg zmB*|4R3ednZNnz?`f|2mle}OegNzv?W>_GL6q6cc9^!Pq%njb8kf&M$bP{<>&Dc%? z3Q?{_vlZ@#N$rxh_sh2uUNZZ;D!z{(B`D=Qzik#r41vq57-q3BESW-)!){QuMsz)D zm>w{7e64NW22$qVs&|m>bpEe*FjCt0Lq(5ELM=0{ZgkYV>rV1_DeEbYAZxqJd(6sz zMS~Egu#@cle`DtG^($lJSMO_Pew%?I5N=}TC1d8d$g67P`V&sS#uICYAE9o&o01lD zku-jFO;5 zYR5yZUDK<2g{)5I#dhSvR&InVM>{H4;%dEa+tQR6*p&sV$?WNui~^XcSmQB<>#Ypm zqhhNQ&ruHBckM*(vI*I`!%4Dq{F?Abkan2wAMOYmIq;v-STd~|*dfqDH%alJHC?aV z%k&CpwcN(T-y;maN3aQ6;rEpxn%#j^_|JL0A6S*T^&2_K$e^JC=^5p*1tk8#7RI*f z*qFPwXXzXuEm#AI#M;~xuj znOxi}JxK<-9DI^qw0;bNz;5$_pzvShgWIU-_M!=yh;9V;L|Jbd=HJzzgv72myqnNF zo-Hf(Ed_fNn1VsMT?&S`CN7VAfkb1vtBT=!%J{Z|t<`1Uq6^>9I3*g(wmgW#I@V(o zPWrc1g|d3K@r?4A%&6nI>;D4Eq4BOw1yzgblFEd;s&kN@2nFSzWiV9Mq4*7r2R?nFxA?f)xA%r)Kl*|&rLAam5!UWuD&oYtn5T9mEzilhgJ~R zmssg-d2mfnGw0^+i4P909Sr&8^tA7b%_qc0IIJ~JX0)`7lNW%fR(0PE=cumRF@YRX zx}{H+Z#;f7b}jfmmJe2g2%GLRe&A0Cj~69`5I}!Tk?L zWQK6JT|8I!&F;r}my<>%z7ZbhWcv&+P-KJ`;3sS(1Z8b6fJL6o0_-M1gkw_1yKpI{X%1l3FVOBO=xXk3BCQk1MmcM!DKIk}n zoOV(pnzp5(j+xrcX(m9&35O-xHXS$c>yG?{O0U$p8Z7)-Ubke1BrR$%ry@bL;-j7Ot(NMj8?^%0Dd z5tky*C&}{JGiU1b!e8smX_^`*>!URyOJ{T?wC=4t^W4)fJ^j2@VJ_&VB`lXapQ#Dn z8MgD+zuUFSAwcoYqjs}__g;TjX+j6@)GP5lbt1Hr6XozZQvAM9GGjiF%XXc(xjKzRnGF{#EqZ(uR!!e2Z6qcj8;bOtaHy&2b;&oUQawBrW_cWywc|OJ$gO zev;t)D9~CD|F!}{hrg&;OTk}KAkiZH1qG(tx}eyX6nv!M zwt`y9 zUg%$OcWK}8gQH@JgPdwP#)%gHyBF2jiNfB}q2D|7(4ixT9zGOk$AcPnd-{E`A3WX3 hSN>O0yJCAu)1h8nx_Fc_4_hqFDUws_-{0T+zX7FOCaVAd literal 0 HcmV?d00001 diff --git a/contrib/site-packages/Pyro4/utils/__init__.py b/contrib/site-packages/Pyro4/utils/__init__.py new file mode 100644 index 00000000..3c55dacc --- /dev/null +++ b/contrib/site-packages/Pyro4/utils/__init__.py @@ -0,0 +1 @@ +# just to make this a package. diff --git a/contrib/site-packages/Pyro4/utils/__init__.pyo b/contrib/site-packages/Pyro4/utils/__init__.pyo new file mode 100644 index 0000000000000000000000000000000000000000..6b1df4e9eb29da1927bce9ca98fcbf5b412134d3 GIT binary patch literal 198 zcmZSn%*z$`cwta70~9a#hLke#rj5i=K49AN%{p~6(C-5 zW=X1UL1J=tVtOh_XHmY1erZW&PO*M`d}dx|NqoE>P%#J4PMh5Pl+v73JCMtYftUdR DnbI@~ literal 0 HcmV?d00001 diff --git a/contrib/site-packages/Pyro4/utils/flame.py b/contrib/site-packages/Pyro4/utils/flame.py new file mode 100644 index 00000000..9672652f --- /dev/null +++ b/contrib/site-packages/Pyro4/utils/flame.py @@ -0,0 +1,299 @@ +""" +Pyro FLAME: Foreign Location Automatic Module Exposer. +Easy but potentially very dangerous way of exposing remote modules and builtins. +Flame requires the pickle serializer to be used. + +Pyro - Python Remote Objects. Copyright by Irmen de Jong (irmen@razorvine.net). +""" + +from __future__ import with_statement +import sys +import types +import code +import Pyro4.core +import Pyro4.util +import Pyro4.constants +import Pyro4.errors +try: + import importlib +except ImportError: + importlib = None +try: + import builtins +except ImportError: + import __builtin__ as builtins +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO + +__all__ = ["connect", "start", "createModule", "Flame"] + + +# Exec is a statement in Py2, a function in Py3 +# Workaround as written by Ned Batchelder on his blog. +if sys.version_info > (3, 0): + def exec_function(source, filename, global_map): + source=fixExecSourceNewlines(source) + exec(compile(source, filename, "exec"), global_map) +else: + # OK, this is pretty gross. In Py2, exec was a statement, but that will + # be a syntax error if we try to put it in a Py3 file, even if it isn't + # executed. So hide it inside an evaluated string literal instead. + eval(compile("""\ +def exec_function(source, filename, global_map): + source=fixExecSourceNewlines(source) + exec compile(source, filename, "exec") in global_map +""", + "", "exec" + )) + +def fixExecSourceNewlines(source): + if sys.version_info < (2,7) or sys.version_info[:2] in ((3,0), (3,1)): + # for python versions prior to 2.7 (and 3.0/3.1), compile is kinda picky. + # it needs unix type newlines and a trailing newline to work correctly. + source = source.replace("\r\n", "\n") + source = source.rstrip() + "\n" + # remove trailing whitespace that might cause IndentationErrors + source = source.rstrip() + return source + + +class FlameModule(object): + """Proxy to a remote module.""" + def __init__(self, flameserver, module): + # store a proxy to the flameserver regardless of autoproxy setting + self.flameserver = Pyro4.core.Proxy(flameserver._pyroDaemon.uriFor(flameserver)) + self.module = module + + def __getattr__(self, item): + if item in ("__getnewargs__", "__getinitargs__"): + raise AttributeError(item) + return Pyro4.core._RemoteMethod(self.__invoke, "%s.%s" % (self.module, item)) + + def __getstate__(self): + return self.__dict__ + + def __setstate__(self, args): + self.__dict__ = args + + def __invoke(self, module, args, kwargs): + return self.flameserver._invokeModule(module, args, kwargs) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.flameserver._pyroRelease() + + def __repr__(self): + return "<%s.%s at 0x%x, module '%s' at %s>" % (self.__class__.__module__, self.__class__.__name__, + id(self), self.module, self.flameserver._pyroUri.location) + + +class FlameBuiltin(object): + """Proxy to a remote builtin function.""" + def __init__(self, flameserver, builtin): + # store a proxy to the flameserver regardless of autoproxy setting + self.flameserver = Pyro4.core.Proxy(flameserver._pyroDaemon.uriFor(flameserver)) + self.builtin = builtin + + def __call__(self, *args, **kwargs): + return self.flameserver._invokeBuiltin(self.builtin, args, kwargs) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.flameserver._pyroRelease() + + def __repr__(self): + return "<%s.%s at 0x%x, builtin '%s' at %s>" % (self.__class__.__module__, self.__class__.__name__, + id(self), self.builtin, self.flameserver._pyroUri.location) + + +class RemoteInteractiveConsole(object): + """Proxy to a remote interactive console.""" + + class LineSendingConsole(code.InteractiveConsole): + """makes sure the lines are sent to the remote console""" + def __init__(self, remoteconsole): + code.InteractiveConsole.__init__(self, filename="") + self.remoteconsole = remoteconsole + + def push(self, line): + output, more = self.remoteconsole.push_and_get_output(line) + if output: + sys.stdout.write(output) + return more + + def __init__(self, remoteconsoleuri): + # store a proxy to the console regardless of autoproxy setting + self.remoteconsole = Pyro4.core.Proxy(remoteconsoleuri) + + def interact(self): + console = self.LineSendingConsole(self.remoteconsole) + console.interact(banner=self.remoteconsole.get_banner()) + print("(Remote session ended)") + + def close(self): + self.remoteconsole.terminate() + self.remoteconsole._pyroRelease() + + def __repr__(self): + return "<%s.%s at 0x%x, for %s>" % (self.__class__.__module__, self.__class__.__name__, + id(self), self.remoteconsole._pyroUri.location) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + +class InteractiveConsole(code.InteractiveConsole): + """Interactive console wrapper that saves output written to stdout so it can be returned as value""" + def push_and_get_output(self, line): + output, more = "", False + stdout_save = sys.stdout + try: + sys.stdout = StringIO() + more = self.push(line) + output = sys.stdout.getvalue() + sys.stdout.close() + finally: + sys.stdout = stdout_save + return output, more + + def get_banner(self): + return self.banner # custom banner string, set by Pyro daemon + + def write(self, data): + sys.stdout.write(data) # stdout instead of stderr + + def terminate(self): + self._pyroDaemon.unregister(self) + self.resetbuffer() + + +class Flame(object): + """ + The actual FLAME server logic. + Usually created by using :py:meth:`Pyro4.core.Daemon.startFlame`. + Be *very* cautious before starting this: it allows the clients full access to everything on your system. + """ + def __init__(self): + if "pickle" not in Pyro4.config.SERIALIZERS_ACCEPTED: + raise RuntimeError("flame requires the pickle serializer to be enabled") + + def module(self, name): + """import a module on the server given by the module name and returns a proxy to it""" + if importlib: + importlib.import_module(name) + else: + __import__(name) + return FlameModule(self, name) + + def builtin(self, name): + """returns a proxy to the given builtin on the server""" + return FlameBuiltin(self, name) + + def execute(self, code): + """execute a piece of code""" + exec_function(code, "", globals()) + + def evaluate(self, expression): + """evaluate an expression and return its result""" + return eval(expression) + + def sendmodule(self, modulename, modulesource): + """ + Send the source of a module to the server and make the server load it. + Note that you still have to actually ``import`` it on the server to access it. + Sending a module again will replace the previous one with the new. + """ + createModule(modulename, modulesource) + + def getmodule(self, modulename): + """obtain the source code from a module on the server""" + import inspect + module = __import__(modulename, globals={}, locals={}) + return inspect.getsource(module) + + def sendfile(self, filename, filedata): + """store a new file on the server""" + import os, stat + with open(filename, "wb") as targetfile: + os.chmod(filename, stat.S_IRUSR | stat.S_IWUSR) # readable/writable by owner only + targetfile.write(filedata) + + def getfile(self, filename): + """read any accessible file from the server""" + with open(filename, "rb") as file: + return file.read() + + def console(self): + """get a proxy for a remote interactive console session""" + console = InteractiveConsole(filename="") + uri = self._pyroDaemon.register(console) + console.banner = "Python %s on %s\n(Remote console on %s)" % (sys.version, sys.platform, uri.location) + return RemoteInteractiveConsole(uri) + + def _invokeBuiltin(self, builtin, args, kwargs): + return getattr(builtins, builtin)(*args, **kwargs) + + def _invokeModule(self, dottedname, args, kwargs): + # dottedname is something like "os.path.walk" so strip off the module name + modulename, dottedname = dottedname.split('.', 1) + module = sys.modules[modulename] + # we override the DOTTEDNAMES setting here because this safeguard makes no sense + # with the Flame server (if enabled it already allows full access to anything): + method = Pyro4.util.resolveDottedAttribute(module, dottedname, True) + return method(*args, **kwargs) + + +def createModule(name, source, filename="", namespace=None): + """ + Utility function to create a new module with the given name (dotted notation allowed), directly from the source string. + Adds it to sys.modules, and returns the new module object. + If you provide a namespace dict (such as ``globals()``), it will import the module into that namespace too. + """ + path = "" + components = name.split('.') + module = types.ModuleType("pyro-flame-module-context") + for component in components: + # build the module hierarchy. + path += '.' + component + real_path = path[1:] + if real_path in sys.modules: + # use already loaded modules instead of overwriting them + module = sys.modules[real_path] + else: + setattr(module, component, types.ModuleType(real_path)) + module = getattr(module, component) + sys.modules[real_path] = module + exec_function(source, filename, module.__dict__) + if namespace is not None: + namespace[components[0]] = __import__(name) + return module + + +def start(daemon): + """ + Create and register a Flame server in the given daemon. + Be *very* cautious before starting this: it allows the clients full access to everything on your system. + """ + if Pyro4.config.FLAME_ENABLED: + return daemon.register(Flame(), Pyro4.constants.FLAME_NAME) + else: + raise Pyro4.errors.SecurityError("Flame is disabled in the server configuration") + + +def connect(location): + """ + Connect to a Flame server on the given location, for instance localhost:9999 or ./u:unixsock + This is just a convenience function to creates an appropriate Pyro proxy. + """ + proxy = Pyro4.core.Proxy("PYRO:%s@%s" % (Pyro4.constants.FLAME_NAME, location)) + proxy._pyroBind() + return proxy diff --git a/contrib/site-packages/Pyro4/utils/flame.pyo b/contrib/site-packages/Pyro4/utils/flame.pyo new file mode 100644 index 0000000000000000000000000000000000000000..218c6f1f70bf0fae97d705f672284a84c0f9ab08 GIT binary patch literal 17806 zcmd5^O>7)lUVqiyZrdH_b23gmA3HT4+3CzC?b&5u*lafB%V9Dko=~2d%$S)ix4X)A z#qF-%deydjNFdq_2nmTr8i@nq!VL+D3tTvGLL4}7;Dito9FVwh195@F^85Y&_0e_$ zT8&n2JFe&VUcIV%@BjV&|Nl$%KaVzk@UMUDbxig%fxlND8)N_3F(xo(8Kq-_is?FL zt74Wb#$mf^x>d7Wm3yey%yKQSPnhM2yk0lUlX-n=q&_`TZ?HZ*kq-ZTHahU-4kZ(q**>`ocq03 zi)a2Pe+C0i6IRUwyzFJOSu@FbCpcoh1bm)1y;&3D_lyZ< zOmNf$$IO$edEx+2$7RG>6Pz&NoLR%nPO|5TF`q8KV1iS5?{h`(izYao_dZ|rzGMP? zV8T3jSzdCcxbZS)m%aLI(fbV(%n?Pw3+79p>=hHd$a~AK30{)QtGv6^{04Asj{?t) zd&4AlZ{4|ecj1!j-inhjTJO1c;!Zn_;+}hLkj7i6cHFyhFzANv!frpdVbW?WwC&Jc z9i(nQPQzXrwY%M+yB#J&H)!|P!z3P9cc(pc<25(rGf{8dO~Neqt%gqhqI++cZeXgO zzWjsL2Vp0*E!Vvf_lHTezLC1CL-%&F753a9bl;CLk!Hl^)ujDJoNPzEu+i{;sDqFeo_8#ejXw2s2cn3OA8!;K!`pEl4~ z@5Za`uD{jpH_cfK2~&i4gVY}qIcFZSD=&Gvf8CzJS{`9T_W?fe?apoNh~@j))lUmNr~B*r(; zVYy)A%sBQd{0%}9*T|5jF6n}sujhhWF6R8DhKnB^^8IJRDSQ6B4CI^LPOLzC`o*vD z{%hyms5kPBhCPmXUEVXPD;|N}AzASvo!oegrF@P`YRm&*c*<;|qnS4y;|9|vXb|I@ zb(7pBB|HH3MJhe1kUZD}Spy(UnlotkM!NL&Qu7GDuo#h2B2sEeeGw;OdE;oqvfdB^ zDu-&5lp@qR(&XqSv5@kv9aRJh%E{G-F7DwI!i40r7!H`P4jVFW1=dL zaQHb~mV|DcX7!ZwTxG^d2`XVxW!Vnz@`uY6_WT$Xu;((cg%Y@MM4m#_R-^(YSd|Kt zM@=d~)`V1`Na|9VBC@{3<3R3|R2s%xY7!=P4#mAB-W^iLwfBn3Rv|42b0MAMyc;#<;*xeQ+d_tkn(vI zym$o-irSj++lvIP!tzHBIV%N#D}*t*PB6_10PK)OXTgz6O}=YU^oH-RhiNa|X(#K} z_ftv)>EcYZ&vNiN+d5~3vn;jE-lx2m(AA{ikXK$y!9vis;X;zciFXS9-gCT3De0@~ zyBk6e2I*<`jF2)RupMuPiVm*YBdrPMA%(Zbi6L^Gth0?uPYn(8c!Yp#mx`%p>T&^jsN)}OByEps?Xlv>C4Vc3S?yd^a9 zCFJPiGGqxSQ7KUP1nUdR^_18XlBe46!fx2MVTnB9&v3Wnr^9}j^1Ezox4Q#vr3oyB zRTzWgM`*Z9WkFq^LnB8f$xfj0^Qb`i7XZmB6&ggStMa8T77g*BZXQgUwFxtune3Jf zeOY9`+fLoLchBu!$aX~To9FDC>^x_$G)Z6rE8p*Q+m?!iCX4T@zC}|>kXAby#6uJ) zn+P+>U83WoB$A=D+i6B04(ij%!%774eTunfhW+!77_Hv7(dE63LX1h@58{sRdoQr} zRTi(aILG1@7Ux;eB=U%zCa#gvv6w|eU)SAl2gBrt z#CH8L9{e6E;n%e`VN#M>ErIm3Ntx}q-F#d1vo=%86!lBk>!v7m1Gyi%j%zKN7;Im4U-(u|oi;KL0 zON)c2Op3NmT>CNeGgC^hub>nxNHe4G%vhDB&+--3; z?BJgKP|}D(`5cvf3WZMXns}APgYkr}X*e&`ivgd3dNJUZGJ8(`eg}ngY)crFrRH6Iy%)f+Y9C5rr-*KIml(*I1 zg!9-Al2AOz;{9~nXjxdgbgj^@ocW=%sTAG-a=ys9*HM8702>xMSH#?dm*)wB5!p5k zAEyb-uP%Bt%hc_s@6BF&MZIKvX(tXsS?|-B63UaE&z^_N9iR70c)>Vj4_RZEOXNz7 zGpid|HfVDeCUL11#R2shoRs(#zjg9$1N$`dP!W#C0Y_pNm~V?heH% z`-Cw+1Tib-m){EtnIR;)+D70gu{4dFo(8XMXz50GAriyjyaJABQ05@*%@NaYFDmCi~mCGJoDFXYLvPJcc0^iUc7TBWplN>~%8`?g|9D51{V0q8{RF zYO#^Zgp9_xUXe(hE@Gp4kyYwyyX}Mfvhug)G@X z1F{7Ax1z?D#;G3F%opFII}#L#9$#$OB0310EDL8%Z~z|6nspJn_XE&xEqm@0_0}WC zl_wnFi>8br-fDMkD8`w04L3ZYaT+h;Q-J*@6BqhWV@I4GH#qbriv-?)1kFuoTJXPQEQA%^JQ=~cNu5x=Wssse@q5E)be;IiC6I{8x zq7jo{vVzm-*k{0v`uD52J%ZrtXpYz#lzc%uZI2UYoYJ4-%2!AP@Zu_Eg?o$)AdCAC z;_TT*2HllnTn&0jxE@*98bYDSNea`|!P;7w91hIh!9@QISEfZzIKr0uW%gB!@JIw6 z;pFk55?N9^crK#~`9oZigCQG>8M?+h=#n2|Pvr3zsDxkF3;zG)Tp+oIa{UNXY*<0^ zo@VqRg>OcBj?tsRO{e`Q6p{|~Ash#AvJBc?O_yU@8gd}qZoD3KS~BD#i@QjJ)8sEC zw+)f1#>BTv{oy4fS8ZHck*tdrGcH=1UeS`AF^LAO=v%Ib?pt{A@GWQp-;xGBLCZ-9)$Elq|v9Nb7_=r3_)+<2-C z^HYL~9J+`KByBRA$y@l6D2jRwdPlN`#t_+{$|WOMMwg?lKC*Azb{;4MCh5n@C~zH) zKbQ>2973Ol2VI5*CBah72qas&{XDxUN(JN+UDP)tIZu30+R8$VS-dgp^ChaaPxkXG zr8qfwNkSj!UlB-CC`_g9oS>E_Df22IPeumhA@uh^mgM>j@DPd_NHsi1#wo$A^hUpI z2@LNuHgRcj@PMHE{cmuo608&;pn?|SYc~ zcO&p1i{zKk2}9Qk4+b?gfZ+0$eQZEC*oI8iQP z31CU*^Y@sUGC^I01J_ZTHGz>3_sz)ai-5;_teE1qIq%R0(a^RSJ0|{bSoLmH;3R z*86eGJVJ^oqI8I&z@=Q13}2KJ<~lj^p?C<%rm!OJ19`vtiHLJT$a2$TQyPtJ7j!HM z@q1ZDaUh99JFB{-r&NrsNHV7ONhR*XswApA8(5!UXUV_qeYE7|_4}yH?n1Io6gMJP zxnd^bbBE|XK*M7UTX=?fuwlxZ$h`3@bzH&MD^wH6w^TE?yA#O;idU zN(7yx_XG$~=k?ME4)~npXyO$vct}<=#R@jm1YJH}iW>!BO)Du zBxZIC7_xK{tJ^(s1k!5t-eB=2iYBgc+0Wm_r~d<2?nL5xiAVmGHIt07T1c^sk}Kvj z#J6h(^sN3<$mco8Yo%inu^bP8dV`()sBRwOw}ca6xrpbCX~|if(xqzq-6)H9a{`$) zAxU+@`8*|+5(BV3gb%q3x4=5o*N<=Lm+Er%oN|vZ6xSNZ$KI-tt%Vg(aU)5K7;`+7<>}x(0B@uKw`i{ zw>fQ+e;|co@^}|IF-asG&0S`ONF7!tnZUjcPEijVs6R@FMe-``e)YCN0npA-IF&ZM zZsbY#Zc}-{?ZpsEjIxXK3~}}gZh#}FofKP)QY)(=%ad!pb=?JA3j)iH2jewE+sZie z!X7UR75d0DVmXYekKbOS_l+8BJA!qL>0(kAXCOm-zLUC5JLqgM*t4>dZ>P_%tYC8Z zq-XWBs1D2*`Pl2tG;yrh;D-=Uhfh*vTU;Yd0170~4Q_ z8b?rizkxz5KBntiksf#D#O{Y^Yb@Y5IYcm{0^a*|KFHmV_Zn|eY%XFiB?i1|OnOM< z^q(3rq~T)HGQQtVH{@eHa*1c*%M=b5`@iUu7ObMK&+#QMA3Pg#jX%L9rnafqabW4V zbF^~IX*kC#Cn|GjokWt)jB}=p8x<;)1^`s3dRUWl!n?46YUYXdi)jyLTH>Hd@(g$! z32DaY0Qg~e;v?=0N(c6g6MHKUXs_AVeEp(P!; zh|}$&;zFLDv+|B5xVjL49m;_^_25vZDrm+@S48a`y>Eq+`4#9_R>nWRaaWckB#6=HLPrG%mG}8 zEZMPdV6%AXyZCW&t2IBkH0VXUHtuZ72R=lQ!o}}{fu*Sqw8Bk|7zLir&mNTGoBtbt zi%5AA_i@-C&L9R#)Vxv48fRG>kS9MMxb)bRN0scLKzo+r{bmi2tA9=_CIOO)KX{ literal 0 HcmV?d00001 diff --git a/contrib/site-packages/Pyro4/utils/flameserver.py b/contrib/site-packages/Pyro4/utils/flameserver.py new file mode 100644 index 00000000..f5bc9226 --- /dev/null +++ b/contrib/site-packages/Pyro4/utils/flameserver.py @@ -0,0 +1,55 @@ +""" +Pyro FLAME: Foreign Location Automatic Module Exposer. +Easy but potentially very dangerous way of exposing remote modules and builtins. +This is the commandline server. + +You can start this module as a script from the command line, to easily get a +flame server running: + + :command:`python -m Pyro4.utils.flameserver` + +Pyro - Python Remote Objects. Copyright by Irmen de Jong (irmen@razorvine.net). +""" + +import sys +import Pyro4.utils.flame +import Pyro4.core + + +def main(args, returnWithoutLooping=False): + from optparse import OptionParser + parser = OptionParser() + parser.add_option("-H", "--host", default="localhost", help="hostname to bind server on (default=localhost)") + parser.add_option("-p", "--port", type="int", default=0, help="port to bind server on") + parser.add_option("-u", "--unixsocket", help="Unix domain socket name to bind server on") + parser.add_option("-q", "--quiet", action="store_true", default=False, help="don't output anything") + parser.add_option("-k", "--key", help="the HMAC key to use") + options, args = parser.parse_args(args) + + if not options.quiet: + print("Starting Pyro Flame server.") + + hmac = (options.key or "").encode("utf-8") + if not hmac: + print("Warning: HMAC key not set. Anyone can connect to this server!") + Pyro4.config.HMAC_KEY = hmac or Pyro4.config.HMAC_KEY + if not options.quiet and Pyro4.config.HMAC_KEY: + print("HMAC_KEY set to: %s" % Pyro4.config.HMAC_KEY) + + Pyro4.config.SERIALIZERS_ACCEPTED = set(["pickle"]) # flame requires pickle serializer + + daemon = Pyro4.core.Daemon(host=options.host, port=options.port, unixsocket=options.unixsocket) + uri = Pyro4.utils.flame.start(daemon) + if not options.quiet: + print("server uri: %s" % uri) + print("server is running.") + + if returnWithoutLooping: + return daemon, uri # for unit testing + else: + daemon.requestLoop() + daemon.close() + return 0 + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/contrib/site-packages/Pyro4/utils/flameserver.pyo b/contrib/site-packages/Pyro4/utils/flameserver.pyo new file mode 100644 index 0000000000000000000000000000000000000000..2bc426f342f2832641b47214607a3db9d35b1817 GIT binary patch literal 2471 zcmc&${chV-6hC&_wDr<`bZa-*V0RnSGDIAK5TJ^oRV8JE?nCKdT7~dj-<#B_V|({r zZwqSw8V|r5AfAB7;13>v2jB@14*%CAjs55m1rH~>5F49o_j;>BY!q6h!(40@No><17i*O*#vl`MuaH%y zMDJx;7^TCY7n_OLsjMgqt8$yhSvC>Rm7WM0=R>87%7_>7L=*#|SSQVgLaQ-^#n?d^ z5$6&JX=c;hgu$cT)Chd-t`bQx9^*7ib0uI6uwf89DJqe~xiB`?7GjXCkwgqEVG^B| zRt$78K8_(6y^=jg+* zvT0^Qr&#;+G^n%Jfq)b3*YQ2r8L7mYP>A)Soal78YsJn)+|^^13#r7H1#;0&`SG@n zzc2K2xESWj-VB5EZ+z}2KI$`%E5NnRwfYscnzjMa*rNlVynUZu5gjyWzd<9PTLB$_ zV#_8;bi+Ko@_lc5hAs$5o*>zB5{pjaBqSP#5~oO>cHA@IdWYO4l4nVtBe~3a7AAna za$N5`$qSDA#%sM9+9z(eZ+$wLp#!4*85+%!`Hke8^u--CAv2m({!a2DjevNG#Sdm_ zA5wpkd}}H-OC@=Bo|||SP)$U^6t1P;KRN>~-=0c9bj4Y*i1#uLUF%PkTTW(?Mkh%M z2ZXqT%yrg8l9v4rt@%aYTNwe$!oMf$A;uPQ*S-ezjzP?dw?MW zHCpBhF>i#Nfs%_DzUwIHeOKr?vz*g7GyTg~a|yF|>{?{@b}4<(c6qJDq1$ zs;uSUpHe(_}AGF7>Y~>WaZ}A4};;cFHY7QR$S` z&C(GI+i<$Z!uENv(eFe>X-jUYK4@}zjHQgg<+O5|HsTE4upuWG#d>JmL0tsy0;J8k zsq&Z{ZcWv|TcCWt`t*x*39zN=Q?jlW4PKWS$ zjHQ|P*@gZIzHuu#D{kL9%I#c7x!sSkCerE|dckfMh=wCcGQ5}C9JIrCfv^WKQ|bJ1 ziUnOcX_UN92X~l@TD$S8vtsHxjL~|0E}eHSB;>pe4!N7rpt?}$x^e1mraL;;lkSGb z^t~wbp6S-_gm7@k>3)OBM&3>LgYe^SmhN=x7Ysj{)T&MyCwuWwu_j(OT^D}$*y|>Q zJkKZ36F0W`2siRhdkfwr@2a=ru4RA0oA)nxO>f@20N#?n?7!~m^e-_%E3&i*Oz`ZxHXhUZ}i5c{lN lruA|`U7jDM@qf)FJNaq-AH-+ebZ&_?{g#KnM(bM3`v*>ndO!dG literal 0 HcmV?d00001 diff --git a/contrib/site-packages/paramiko/__init__.py b/contrib/site-packages/paramiko/__init__.py new file mode 100644 index 00000000..b1ef0e6f --- /dev/null +++ b/contrib/site-packages/paramiko/__init__.py @@ -0,0 +1,143 @@ +# Copyright (C) 2003-2011 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +I{Paramiko} (a combination of the esperanto words for "paranoid" and "friend") +is a module for python 2.5 or greater that implements the SSH2 protocol for +secure (encrypted and authenticated) connections to remote machines. Unlike +SSL (aka TLS), the SSH2 protocol does not require hierarchical certificates +signed by a powerful central authority. You may know SSH2 as the protocol that +replaced C{telnet} and C{rsh} for secure access to remote shells, but the +protocol also includes the ability to open arbitrary channels to remote +services across an encrypted tunnel. (This is how C{sftp} works, for example.) + +The high-level client API starts with creation of an L{SSHClient} object. +For more direct control, pass a socket (or socket-like object) to a +L{Transport}, and use L{start_server } or +L{start_client } to negoatite +with the remote host as either a server or client. As a client, you are +responsible for authenticating using a password or private key, and checking +the server's host key. I{(Key signature and verification is done by paramiko, +but you will need to provide private keys and check that the content of a +public key matches what you expected to see.)} As a server, you are +responsible for deciding which users, passwords, and keys to allow, and what +kind of channels to allow. + +Once you have finished, either side may request flow-controlled L{Channel}s to +the other side, which are python objects that act like sockets, but send and +receive data over the encrypted session. + +Paramiko is written entirely in python (no C or platform-dependent code) and is +released under the GNU Lesser General Public License (LGPL). + +Website: U{https://github.com/paramiko/paramiko/} + +Mailing list: U{paramiko@librelist.com} +""" + +import sys + +if sys.version_info < (2, 5): + raise RuntimeError('You need python 2.5+ for this module.') + + +__author__ = "Jeff Forcier " +__version__ = "1.12.0" +__license__ = "GNU Lesser General Public License (LGPL)" + + +from transport import SecurityOptions, Transport +from client import SSHClient, MissingHostKeyPolicy, AutoAddPolicy, RejectPolicy, WarningPolicy +from auth_handler import AuthHandler +from channel import Channel, ChannelFile +from ssh_exception import SSHException, PasswordRequiredException, \ + BadAuthenticationType, ChannelException, BadHostKeyException, \ + AuthenticationException, ProxyCommandFailure +from server import ServerInterface, SubsystemHandler, InteractiveQuery +from rsakey import RSAKey +from dsskey import DSSKey +from ecdsakey import ECDSAKey +from sftp import SFTPError, BaseSFTP +from sftp_client import SFTP, SFTPClient +from sftp_server import SFTPServer +from sftp_attr import SFTPAttributes +from sftp_handle import SFTPHandle +from sftp_si import SFTPServerInterface +from sftp_file import SFTPFile +from message import Message +from packet import Packetizer +from file import BufferedFile +from agent import Agent, AgentKey +from pkey import PKey +from hostkeys import HostKeys +from config import SSHConfig +from proxy import ProxyCommand + +# fix module names for epydoc +for c in locals().values(): + if issubclass(type(c), type) or type(c).__name__ == 'classobj': + # classobj for exceptions :/ + c.__module__ = __name__ +del c + +from common import AUTH_SUCCESSFUL, AUTH_PARTIALLY_SUCCESSFUL, AUTH_FAILED, \ + OPEN_SUCCEEDED, OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED, OPEN_FAILED_CONNECT_FAILED, \ + OPEN_FAILED_UNKNOWN_CHANNEL_TYPE, OPEN_FAILED_RESOURCE_SHORTAGE + +from sftp import SFTP_OK, SFTP_EOF, SFTP_NO_SUCH_FILE, SFTP_PERMISSION_DENIED, SFTP_FAILURE, \ + SFTP_BAD_MESSAGE, SFTP_NO_CONNECTION, SFTP_CONNECTION_LOST, SFTP_OP_UNSUPPORTED + +from common import io_sleep + +__all__ = [ 'Transport', + 'SSHClient', + 'MissingHostKeyPolicy', + 'AutoAddPolicy', + 'RejectPolicy', + 'WarningPolicy', + 'SecurityOptions', + 'SubsystemHandler', + 'Channel', + 'PKey', + 'RSAKey', + 'DSSKey', + 'Message', + 'SSHException', + 'AuthenticationException', + 'PasswordRequiredException', + 'BadAuthenticationType', + 'ChannelException', + 'BadHostKeyException', + 'ProxyCommand', + 'ProxyCommandFailure', + 'SFTP', + 'SFTPFile', + 'SFTPHandle', + 'SFTPClient', + 'SFTPServer', + 'SFTPError', + 'SFTPAttributes', + 'SFTPServerInterface', + 'ServerInterface', + 'BufferedFile', + 'Agent', + 'AgentKey', + 'HostKeys', + 'SSHConfig', + 'util', + 'io_sleep' ] diff --git a/contrib/site-packages/paramiko/_winapi.py b/contrib/site-packages/paramiko/_winapi.py new file mode 100644 index 00000000..f141b005 --- /dev/null +++ b/contrib/site-packages/paramiko/_winapi.py @@ -0,0 +1,269 @@ +""" +Windows API functions implemented as ctypes functions and classes as found +in jaraco.windows (2.10). + +If you encounter issues with this module, please consider reporting the issues +in jaraco.windows and asking the author to port the fixes back here. +""" + +import ctypes +import ctypes.wintypes +import __builtin__ + +###################### +# jaraco.windows.error + +def format_system_message(errno): + """ + Call FormatMessage with a system error number to retrieve + the descriptive error message. + """ + # first some flags used by FormatMessageW + ALLOCATE_BUFFER = 0x100 + ARGUMENT_ARRAY = 0x2000 + FROM_HMODULE = 0x800 + FROM_STRING = 0x400 + FROM_SYSTEM = 0x1000 + IGNORE_INSERTS = 0x200 + + # Let FormatMessageW allocate the buffer (we'll free it below) + # Also, let it know we want a system error message. + flags = ALLOCATE_BUFFER | FROM_SYSTEM + source = None + message_id = errno + language_id = 0 + result_buffer = ctypes.wintypes.LPWSTR() + buffer_size = 0 + arguments = None + bytes = ctypes.windll.kernel32.FormatMessageW( + flags, + source, + message_id, + language_id, + ctypes.byref(result_buffer), + buffer_size, + arguments, + ) + # note the following will cause an infinite loop if GetLastError + # repeatedly returns an error that cannot be formatted, although + # this should not happen. + handle_nonzero_success(bytes) + message = result_buffer.value + ctypes.windll.kernel32.LocalFree(result_buffer) + return message + + +class WindowsError(__builtin__.WindowsError): + "more info about errors at http://msdn.microsoft.com/en-us/library/ms681381(VS.85).aspx" + + def __init__(self, value=None): + if value is None: + value = ctypes.windll.kernel32.GetLastError() + strerror = format_system_message(value) + super(WindowsError, self).__init__(value, strerror) + + @property + def message(self): + return self.strerror + + @property + def code(self): + return self.winerror + + def __str__(self): + return self.message + + def __repr__(self): + return '{self.__class__.__name__}({self.winerror})'.format(**vars()) + +def handle_nonzero_success(result): + if result == 0: + raise WindowsError() + + +##################### +# jaraco.windows.mmap + +CreateFileMapping = ctypes.windll.kernel32.CreateFileMappingW +CreateFileMapping.argtypes = [ + ctypes.wintypes.HANDLE, + ctypes.c_void_p, + ctypes.wintypes.DWORD, + ctypes.wintypes.DWORD, + ctypes.wintypes.DWORD, + ctypes.wintypes.LPWSTR, +] +CreateFileMapping.restype = ctypes.wintypes.HANDLE + +MapViewOfFile = ctypes.windll.kernel32.MapViewOfFile +MapViewOfFile.restype = ctypes.wintypes.HANDLE + +class MemoryMap(object): + """ + A memory map object which can have security attributes overrideden. + """ + def __init__(self, name, length, security_attributes=None): + self.name = name + self.length = length + self.security_attributes = security_attributes + self.pos = 0 + + def __enter__(self): + p_SA = ( + ctypes.byref(self.security_attributes) + if self.security_attributes else None + ) + INVALID_HANDLE_VALUE = -1 + PAGE_READWRITE = 0x4 + FILE_MAP_WRITE = 0x2 + filemap = ctypes.windll.kernel32.CreateFileMappingW( + INVALID_HANDLE_VALUE, p_SA, PAGE_READWRITE, 0, self.length, + unicode(self.name)) + handle_nonzero_success(filemap) + if filemap == INVALID_HANDLE_VALUE: + raise Exception("Failed to create file mapping") + self.filemap = filemap + self.view = MapViewOfFile(filemap, FILE_MAP_WRITE, 0, 0, 0) + return self + + def seek(self, pos): + self.pos = pos + + def write(self, msg): + ctypes.windll.msvcrt.memcpy(self.view + self.pos, msg, len(msg)) + self.pos += len(msg) + + def read(self, n): + """ + Read n bytes from mapped view. + """ + out = ctypes.create_string_buffer(n) + ctypes.windll.msvcrt.memcpy(out, self.view + self.pos, n) + self.pos += n + return out.raw + + def __exit__(self, exc_type, exc_val, tb): + ctypes.windll.kernel32.UnmapViewOfFile(self.view) + ctypes.windll.kernel32.CloseHandle(self.filemap) + +######################### +# jaraco.windows.security + +class TokenInformationClass: + TokenUser = 1 + +class TOKEN_USER(ctypes.Structure): + num = 1 + _fields_ = [ + ('SID', ctypes.c_void_p), + ('ATTRIBUTES', ctypes.wintypes.DWORD), + ] + + +class SECURITY_DESCRIPTOR(ctypes.Structure): + """ + typedef struct _SECURITY_DESCRIPTOR + { + UCHAR Revision; + UCHAR Sbz1; + SECURITY_DESCRIPTOR_CONTROL Control; + PSID Owner; + PSID Group; + PACL Sacl; + PACL Dacl; + } SECURITY_DESCRIPTOR; + """ + SECURITY_DESCRIPTOR_CONTROL = ctypes.wintypes.USHORT + REVISION = 1 + + _fields_ = [ + ('Revision', ctypes.c_ubyte), + ('Sbz1', ctypes.c_ubyte), + ('Control', SECURITY_DESCRIPTOR_CONTROL), + ('Owner', ctypes.c_void_p), + ('Group', ctypes.c_void_p), + ('Sacl', ctypes.c_void_p), + ('Dacl', ctypes.c_void_p), + ] + +class SECURITY_ATTRIBUTES(ctypes.Structure): + """ + typedef struct _SECURITY_ATTRIBUTES { + DWORD nLength; + LPVOID lpSecurityDescriptor; + BOOL bInheritHandle; + } SECURITY_ATTRIBUTES; + """ + _fields_ = [ + ('nLength', ctypes.wintypes.DWORD), + ('lpSecurityDescriptor', ctypes.c_void_p), + ('bInheritHandle', ctypes.wintypes.BOOL), + ] + + def __init__(self, *args, **kwargs): + super(SECURITY_ATTRIBUTES, self).__init__(*args, **kwargs) + self.nLength = ctypes.sizeof(SECURITY_ATTRIBUTES) + + def _get_descriptor(self): + return self._descriptor + def _set_descriptor(self, descriptor): + self._descriptor = descriptor + self.lpSecurityDescriptor = ctypes.addressof(descriptor) + descriptor = property(_get_descriptor, _set_descriptor) + +def GetTokenInformation(token, information_class): + """ + Given a token, get the token information for it. + """ + data_size = ctypes.wintypes.DWORD() + ctypes.windll.advapi32.GetTokenInformation(token, information_class.num, + 0, 0, ctypes.byref(data_size)) + data = ctypes.create_string_buffer(data_size.value) + handle_nonzero_success(ctypes.windll.advapi32.GetTokenInformation(token, + information_class.num, + ctypes.byref(data), ctypes.sizeof(data), + ctypes.byref(data_size))) + return ctypes.cast(data, ctypes.POINTER(TOKEN_USER)).contents + +class TokenAccess: + TOKEN_QUERY = 0x8 + +def OpenProcessToken(proc_handle, access): + result = ctypes.wintypes.HANDLE() + proc_handle = ctypes.wintypes.HANDLE(proc_handle) + handle_nonzero_success(ctypes.windll.advapi32.OpenProcessToken( + proc_handle, access, ctypes.byref(result))) + return result + +def get_current_user(): + """ + Return a TOKEN_USER for the owner of this process. + """ + process = OpenProcessToken( + ctypes.windll.kernel32.GetCurrentProcess(), + TokenAccess.TOKEN_QUERY, + ) + return GetTokenInformation(process, TOKEN_USER) + +def get_security_attributes_for_user(user=None): + """ + Return a SECURITY_ATTRIBUTES structure with the SID set to the + specified user (uses current user if none is specified). + """ + if user is None: + user = get_current_user() + + assert isinstance(user, TOKEN_USER), "user must be TOKEN_USER instance" + + SD = SECURITY_DESCRIPTOR() + SA = SECURITY_ATTRIBUTES() + # by attaching the actual security descriptor, it will be garbage- + # collected with the security attributes + SA.descriptor = SD + SA.bInheritHandle = 1 + + ctypes.windll.advapi32.InitializeSecurityDescriptor(ctypes.byref(SD), + SECURITY_DESCRIPTOR.REVISION) + ctypes.windll.advapi32.SetSecurityDescriptorOwner(ctypes.byref(SD), + user.SID, 0) + return SA diff --git a/contrib/site-packages/paramiko/agent.py b/contrib/site-packages/paramiko/agent.py new file mode 100644 index 00000000..23a5a2e4 --- /dev/null +++ b/contrib/site-packages/paramiko/agent.py @@ -0,0 +1,380 @@ +# Copyright (C) 2003-2007 John Rochester +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +SSH Agent interface for Unix clients. +""" + +import os +import socket +import struct +import sys +import threading +import time +import tempfile +import stat +from select import select + +from paramiko.ssh_exception import SSHException +from paramiko.message import Message +from paramiko.pkey import PKey +from paramiko.channel import Channel +from paramiko.common import io_sleep +from paramiko.util import retry_on_signal + +SSH2_AGENTC_REQUEST_IDENTITIES, SSH2_AGENT_IDENTITIES_ANSWER, \ + SSH2_AGENTC_SIGN_REQUEST, SSH2_AGENT_SIGN_RESPONSE = range(11, 15) + +class AgentSSH(object): + """ + Client interface for using private keys from an SSH agent running on the + local machine. If an SSH agent is running, this class can be used to + connect to it and retreive L{PKey} objects which can be used when + attempting to authenticate to remote SSH servers. + + Because the SSH agent protocol uses environment variables and unix-domain + sockets, this probably doesn't work on Windows. It does work on most + posix platforms though (Linux and MacOS X, for example). + """ + def __init__(self): + self._conn = None + self._keys = () + + def get_keys(self): + """ + Return the list of keys available through the SSH agent, if any. If + no SSH agent was running (or it couldn't be contacted), an empty list + will be returned. + + @return: a list of keys available on the SSH agent + @rtype: tuple of L{AgentKey} + """ + return self._keys + + def _connect(self, conn): + self._conn = conn + ptype, result = self._send_message(chr(SSH2_AGENTC_REQUEST_IDENTITIES)) + if ptype != SSH2_AGENT_IDENTITIES_ANSWER: + raise SSHException('could not get keys from ssh-agent') + keys = [] + for i in range(result.get_int()): + keys.append(AgentKey(self, result.get_string())) + result.get_string() + self._keys = tuple(keys) + + def _close(self): + #self._conn.close() + self._conn = None + self._keys = () + + def _send_message(self, msg): + msg = str(msg) + self._conn.send(struct.pack('>I', len(msg)) + msg) + l = self._read_all(4) + msg = Message(self._read_all(struct.unpack('>I', l)[0])) + return ord(msg.get_byte()), msg + + def _read_all(self, wanted): + result = self._conn.recv(wanted) + while len(result) < wanted: + if len(result) == 0: + raise SSHException('lost ssh-agent') + extra = self._conn.recv(wanted - len(result)) + if len(extra) == 0: + raise SSHException('lost ssh-agent') + result += extra + return result + +class AgentProxyThread(threading.Thread): + """ Class in charge of communication between two chan """ + def __init__(self, agent): + threading.Thread.__init__(self, target=self.run) + self._agent = agent + self._exit = False + + def run(self): + try: + (r,addr) = self.get_connection() + self.__inr = r + self.__addr = addr + self._agent.connect() + self._communicate() + except: + #XXX Not sure what to do here ... raise or pass ? + raise + + def _communicate(self): + import fcntl + oldflags = fcntl.fcntl(self.__inr, fcntl.F_GETFL) + fcntl.fcntl(self.__inr, fcntl.F_SETFL, oldflags | os.O_NONBLOCK) + while not self._exit: + events = select([self._agent._conn, self.__inr], [], [], 0.5) + for fd in events[0]: + if self._agent._conn == fd: + data = self._agent._conn.recv(512) + if len(data) != 0: + self.__inr.send(data) + else: + self._close() + break + elif self.__inr == fd: + data = self.__inr.recv(512) + if len(data) != 0: + self._agent._conn.send(data) + else: + self._close() + break + time.sleep(io_sleep) + + def _close(self): + self._exit = True + self.__inr.close() + self._agent._conn.close() + +class AgentLocalProxy(AgentProxyThread): + """ + Class to be used when wanting to ask a local SSH Agent being + asked from a remote fake agent (so use a unix socket for ex.) + """ + def __init__(self, agent): + AgentProxyThread.__init__(self, agent) + + def get_connection(self): + """ Return a pair of socket object and string address + May Block ! + """ + conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + conn.bind(self._agent._get_filename()) + conn.listen(1) + (r,addr) = conn.accept() + return (r, addr) + except: + raise + return None + +class AgentRemoteProxy(AgentProxyThread): + """ + Class to be used when wanting to ask a remote SSH Agent + """ + def __init__(self, agent, chan): + AgentProxyThread.__init__(self, agent) + self.__chan = chan + + def get_connection(self): + """ + Class to be used when wanting to ask a local SSH Agent being + asked from a remote fake agent (so use a unix socket for ex.) + """ + return (self.__chan, None) + +class AgentClientProxy(object): + """ + Class proxying request as a client: + -> client ask for a request_forward_agent() + -> server creates a proxy and a fake SSH Agent + -> server ask for establishing a connection when needed, + calling the forward_agent_handler at client side. + -> the forward_agent_handler launch a thread for connecting + the remote fake agent and the local agent + -> Communication occurs ... + """ + def __init__(self, chanRemote): + self._conn = None + self.__chanR = chanRemote + self.thread = AgentRemoteProxy(self, chanRemote) + self.thread.start() + + def __del__(self): + self.close() + + def connect(self): + """ + Method automatically called by the run() method of the AgentProxyThread + """ + if ('SSH_AUTH_SOCK' in os.environ) and (sys.platform != 'win32'): + conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + retry_on_signal(lambda: conn.connect(os.environ['SSH_AUTH_SOCK'])) + except: + # probably a dangling env var: the ssh agent is gone + return + elif sys.platform == 'win32': + import win_pageant + if win_pageant.can_talk_to_agent(): + conn = win_pageant.PageantConnection() + else: + return + else: + # no agent support + return + self._conn = conn + + def close(self): + """ + Close the current connection and terminate the agent + Should be called manually + """ + if hasattr(self, "thread"): + self.thread._exit = True + self.thread.join(1000) + if self._conn is not None: + self._conn.close() + +class AgentServerProxy(AgentSSH): + """ + @param t : transport used for the Forward for SSH Agent communication + + @raise SSHException: mostly if we lost the agent + """ + def __init__(self, t): + AgentSSH.__init__(self) + self.__t = t + self._dir = tempfile.mkdtemp('sshproxy') + os.chmod(self._dir, stat.S_IRWXU) + self._file = self._dir + '/sshproxy.ssh' + self.thread = AgentLocalProxy(self) + self.thread.start() + + def __del__(self): + self.close() + + def connect(self): + conn_sock = self.__t.open_forward_agent_channel() + if conn_sock is None: + raise SSHException('lost ssh-agent') + conn_sock.set_name('auth-agent') + self._connect(conn_sock) + + def close(self): + """ + Terminate the agent, clean the files, close connections + Should be called manually + """ + os.remove(self._file) + os.rmdir(self._dir) + self.thread._exit = True + self.thread.join(1000) + self._close() + + def get_env(self): + """ + Helper for the environnement under unix + + @return: the SSH_AUTH_SOCK Environnement variables + @rtype: dict + """ + env = {} + env['SSH_AUTH_SOCK'] = self._get_filename() + return env + + def _get_filename(self): + return self._file + +class AgentRequestHandler(object): + def __init__(self, chanClient): + self._conn = None + self.__chanC = chanClient + chanClient.request_forward_agent(self._forward_agent_handler) + self.__clientProxys = [] + + def _forward_agent_handler(self, chanRemote): + self.__clientProxys.append(AgentClientProxy(chanRemote)) + + def __del__(self): + self.close() + + def close(self): + for p in self.__clientProxys: + p.close() + +class Agent(AgentSSH): + """ + Client interface for using private keys from an SSH agent running on the + local machine. If an SSH agent is running, this class can be used to + connect to it and retreive L{PKey} objects which can be used when + attempting to authenticate to remote SSH servers. + + Because the SSH agent protocol uses environment variables and unix-domain + sockets, this probably doesn't work on Windows. It does work on most + posix platforms though (Linux and MacOS X, for example). + """ + + def __init__(self): + """ + Open a session with the local machine's SSH agent, if one is running. + If no agent is running, initialization will succeed, but L{get_keys} + will return an empty tuple. + + @raise SSHException: if an SSH agent is found, but speaks an + incompatible protocol + """ + AgentSSH.__init__(self) + + if ('SSH_AUTH_SOCK' in os.environ) and (sys.platform != 'win32'): + conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + conn.connect(os.environ['SSH_AUTH_SOCK']) + except: + # probably a dangling env var: the ssh agent is gone + return + elif sys.platform == 'win32': + import win_pageant + if win_pageant.can_talk_to_agent(): + conn = win_pageant.PageantConnection() + else: + return + else: + # no agent support + return + self._connect(conn) + + def close(self): + """ + Close the SSH agent connection. + """ + self._close() + +class AgentKey(PKey): + """ + Private key held in a local SSH agent. This type of key can be used for + authenticating to a remote server (signing). Most other key operations + work as expected. + """ + + def __init__(self, agent, blob): + self.agent = agent + self.blob = blob + self.name = Message(blob).get_string() + + def __str__(self): + return self.blob + + def get_name(self): + return self.name + + def sign_ssh_data(self, rng, data): + msg = Message() + msg.add_byte(chr(SSH2_AGENTC_SIGN_REQUEST)) + msg.add_string(self.blob) + msg.add_string(data) + msg.add_int(0) + ptype, result = self.agent._send_message(msg) + if ptype != SSH2_AGENT_SIGN_RESPONSE: + raise SSHException('key cannot be used for signing') + return result.get_string() diff --git a/contrib/site-packages/paramiko/auth_handler.py b/contrib/site-packages/paramiko/auth_handler.py new file mode 100644 index 00000000..acb7c8b8 --- /dev/null +++ b/contrib/site-packages/paramiko/auth_handler.py @@ -0,0 +1,426 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +L{AuthHandler} +""" + +import threading +import weakref + +# this helps freezing utils +import encodings.utf_8 + +from paramiko.common import * +from paramiko import util +from paramiko.message import Message +from paramiko.ssh_exception import SSHException, AuthenticationException, \ + BadAuthenticationType, PartialAuthentication +from paramiko.server import InteractiveQuery + + +class AuthHandler (object): + """ + Internal class to handle the mechanics of authentication. + """ + + def __init__(self, transport): + self.transport = weakref.proxy(transport) + self.username = None + self.authenticated = False + self.auth_event = None + self.auth_method = '' + self.password = None + self.private_key = None + self.interactive_handler = None + self.submethods = None + # for server mode: + self.auth_username = None + self.auth_fail_count = 0 + + def is_authenticated(self): + return self.authenticated + + def get_username(self): + if self.transport.server_mode: + return self.auth_username + else: + return self.username + + def auth_none(self, username, event): + self.transport.lock.acquire() + try: + self.auth_event = event + self.auth_method = 'none' + self.username = username + self._request_auth() + finally: + self.transport.lock.release() + + def auth_publickey(self, username, key, event): + self.transport.lock.acquire() + try: + self.auth_event = event + self.auth_method = 'publickey' + self.username = username + self.private_key = key + self._request_auth() + finally: + self.transport.lock.release() + + def auth_password(self, username, password, event): + self.transport.lock.acquire() + try: + self.auth_event = event + self.auth_method = 'password' + self.username = username + self.password = password + self._request_auth() + finally: + self.transport.lock.release() + + def auth_interactive(self, username, handler, event, submethods=''): + """ + response_list = handler(title, instructions, prompt_list) + """ + self.transport.lock.acquire() + try: + self.auth_event = event + self.auth_method = 'keyboard-interactive' + self.username = username + self.interactive_handler = handler + self.submethods = submethods + self._request_auth() + finally: + self.transport.lock.release() + + def abort(self): + if self.auth_event is not None: + self.auth_event.set() + + + ### internals... + + + def _request_auth(self): + m = Message() + m.add_byte(chr(MSG_SERVICE_REQUEST)) + m.add_string('ssh-userauth') + self.transport._send_message(m) + + def _disconnect_service_not_available(self): + m = Message() + m.add_byte(chr(MSG_DISCONNECT)) + m.add_int(DISCONNECT_SERVICE_NOT_AVAILABLE) + m.add_string('Service not available') + m.add_string('en') + self.transport._send_message(m) + self.transport.close() + + def _disconnect_no_more_auth(self): + m = Message() + m.add_byte(chr(MSG_DISCONNECT)) + m.add_int(DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE) + m.add_string('No more auth methods available') + m.add_string('en') + self.transport._send_message(m) + self.transport.close() + + def _get_session_blob(self, key, service, username): + m = Message() + m.add_string(self.transport.session_id) + m.add_byte(chr(MSG_USERAUTH_REQUEST)) + m.add_string(username) + m.add_string(service) + m.add_string('publickey') + m.add_boolean(1) + m.add_string(key.get_name()) + m.add_string(str(key)) + return str(m) + + def wait_for_response(self, event): + while True: + event.wait(0.1) + if not self.transport.is_active(): + e = self.transport.get_exception() + if (e is None) or issubclass(e.__class__, EOFError): + e = AuthenticationException('Authentication failed.') + raise e + if event.isSet(): + break + if not self.is_authenticated(): + e = self.transport.get_exception() + if e is None: + e = AuthenticationException('Authentication failed.') + # this is horrible. python Exception isn't yet descended from + # object, so type(e) won't work. :( + if issubclass(e.__class__, PartialAuthentication): + return e.allowed_types + raise e + return [] + + def _parse_service_request(self, m): + service = m.get_string() + if self.transport.server_mode and (service == 'ssh-userauth'): + # accepted + m = Message() + m.add_byte(chr(MSG_SERVICE_ACCEPT)) + m.add_string(service) + self.transport._send_message(m) + return + # dunno this one + self._disconnect_service_not_available() + + def _parse_service_accept(self, m): + service = m.get_string() + if service == 'ssh-userauth': + self.transport._log(DEBUG, 'userauth is OK') + m = Message() + m.add_byte(chr(MSG_USERAUTH_REQUEST)) + m.add_string(self.username) + m.add_string('ssh-connection') + m.add_string(self.auth_method) + if self.auth_method == 'password': + m.add_boolean(False) + password = self.password + if isinstance(password, unicode): + password = password.encode('UTF-8') + m.add_string(password) + elif self.auth_method == 'publickey': + m.add_boolean(True) + m.add_string(self.private_key.get_name()) + m.add_string(str(self.private_key)) + blob = self._get_session_blob(self.private_key, 'ssh-connection', self.username) + sig = self.private_key.sign_ssh_data(self.transport.rng, blob) + m.add_string(str(sig)) + elif self.auth_method == 'keyboard-interactive': + m.add_string('') + m.add_string(self.submethods) + elif self.auth_method == 'none': + pass + else: + raise SSHException('Unknown auth method "%s"' % self.auth_method) + self.transport._send_message(m) + else: + self.transport._log(DEBUG, 'Service request "%s" accepted (?)' % service) + + def _send_auth_result(self, username, method, result): + # okay, send result + m = Message() + if result == AUTH_SUCCESSFUL: + self.transport._log(INFO, 'Auth granted (%s).' % method) + m.add_byte(chr(MSG_USERAUTH_SUCCESS)) + self.authenticated = True + else: + self.transport._log(INFO, 'Auth rejected (%s).' % method) + m.add_byte(chr(MSG_USERAUTH_FAILURE)) + m.add_string(self.transport.server_object.get_allowed_auths(username)) + if result == AUTH_PARTIALLY_SUCCESSFUL: + m.add_boolean(1) + else: + m.add_boolean(0) + self.auth_fail_count += 1 + self.transport._send_message(m) + if self.auth_fail_count >= 10: + self._disconnect_no_more_auth() + if result == AUTH_SUCCESSFUL: + self.transport._auth_trigger() + + def _interactive_query(self, q): + # make interactive query instead of response + m = Message() + m.add_byte(chr(MSG_USERAUTH_INFO_REQUEST)) + m.add_string(q.name) + m.add_string(q.instructions) + m.add_string('') + m.add_int(len(q.prompts)) + for p in q.prompts: + m.add_string(p[0]) + m.add_boolean(p[1]) + self.transport._send_message(m) + + def _parse_userauth_request(self, m): + if not self.transport.server_mode: + # er, uh... what? + m = Message() + m.add_byte(chr(MSG_USERAUTH_FAILURE)) + m.add_string('none') + m.add_boolean(0) + self.transport._send_message(m) + return + if self.authenticated: + # ignore + return + username = m.get_string() + service = m.get_string() + method = m.get_string() + self.transport._log(DEBUG, 'Auth request (type=%s) service=%s, username=%s' % (method, service, username)) + if service != 'ssh-connection': + self._disconnect_service_not_available() + return + if (self.auth_username is not None) and (self.auth_username != username): + self.transport._log(WARNING, 'Auth rejected because the client attempted to change username in mid-flight') + self._disconnect_no_more_auth() + return + self.auth_username = username + + if method == 'none': + result = self.transport.server_object.check_auth_none(username) + elif method == 'password': + changereq = m.get_boolean() + password = m.get_string() + try: + password = password.decode('UTF-8') + except UnicodeError: + # some clients/servers expect non-utf-8 passwords! + # in this case, just return the raw byte string. + pass + if changereq: + # always treated as failure, since we don't support changing passwords, but collect + # the list of valid auth types from the callback anyway + self.transport._log(DEBUG, 'Auth request to change passwords (rejected)') + newpassword = m.get_string() + try: + newpassword = newpassword.decode('UTF-8', 'replace') + except UnicodeError: + pass + result = AUTH_FAILED + else: + result = self.transport.server_object.check_auth_password(username, password) + elif method == 'publickey': + sig_attached = m.get_boolean() + keytype = m.get_string() + keyblob = m.get_string() + try: + key = self.transport._key_info[keytype](Message(keyblob)) + except SSHException, e: + self.transport._log(INFO, 'Auth rejected: public key: %s' % str(e)) + key = None + except: + self.transport._log(INFO, 'Auth rejected: unsupported or mangled public key') + key = None + if key is None: + self._disconnect_no_more_auth() + return + # first check if this key is okay... if not, we can skip the verify + result = self.transport.server_object.check_auth_publickey(username, key) + if result != AUTH_FAILED: + # key is okay, verify it + if not sig_attached: + # client wants to know if this key is acceptable, before it + # signs anything... send special "ok" message + m = Message() + m.add_byte(chr(MSG_USERAUTH_PK_OK)) + m.add_string(keytype) + m.add_string(keyblob) + self.transport._send_message(m) + return + sig = Message(m.get_string()) + blob = self._get_session_blob(key, service, username) + if not key.verify_ssh_sig(blob, sig): + self.transport._log(INFO, 'Auth rejected: invalid signature') + result = AUTH_FAILED + elif method == 'keyboard-interactive': + lang = m.get_string() + submethods = m.get_string() + result = self.transport.server_object.check_auth_interactive(username, submethods) + if isinstance(result, InteractiveQuery): + # make interactive query instead of response + self._interactive_query(result) + return + else: + result = self.transport.server_object.check_auth_none(username) + # okay, send result + self._send_auth_result(username, method, result) + + def _parse_userauth_success(self, m): + self.transport._log(INFO, 'Authentication (%s) successful!' % self.auth_method) + self.authenticated = True + self.transport._auth_trigger() + if self.auth_event != None: + self.auth_event.set() + + def _parse_userauth_failure(self, m): + authlist = m.get_list() + partial = m.get_boolean() + if partial: + self.transport._log(INFO, 'Authentication continues...') + self.transport._log(DEBUG, 'Methods: ' + str(authlist)) + self.transport.saved_exception = PartialAuthentication(authlist) + elif self.auth_method not in authlist: + self.transport._log(DEBUG, 'Authentication type (%s) not permitted.' % self.auth_method) + self.transport._log(DEBUG, 'Allowed methods: ' + str(authlist)) + self.transport.saved_exception = BadAuthenticationType('Bad authentication type', authlist) + else: + self.transport._log(INFO, 'Authentication (%s) failed.' % self.auth_method) + self.authenticated = False + self.username = None + if self.auth_event != None: + self.auth_event.set() + + def _parse_userauth_banner(self, m): + banner = m.get_string() + lang = m.get_string() + self.transport._log(INFO, 'Auth banner: ' + banner) + # who cares. + + def _parse_userauth_info_request(self, m): + if self.auth_method != 'keyboard-interactive': + raise SSHException('Illegal info request from server') + title = m.get_string() + instructions = m.get_string() + m.get_string() # lang + prompts = m.get_int() + prompt_list = [] + for i in range(prompts): + prompt_list.append((m.get_string(), m.get_boolean())) + response_list = self.interactive_handler(title, instructions, prompt_list) + + m = Message() + m.add_byte(chr(MSG_USERAUTH_INFO_RESPONSE)) + m.add_int(len(response_list)) + for r in response_list: + m.add_string(r) + self.transport._send_message(m) + + def _parse_userauth_info_response(self, m): + if not self.transport.server_mode: + raise SSHException('Illegal info response from server') + n = m.get_int() + responses = [] + for i in range(n): + responses.append(m.get_string()) + result = self.transport.server_object.check_auth_interactive_response(responses) + if isinstance(type(result), InteractiveQuery): + # make interactive query instead of response + self._interactive_query(result) + return + self._send_auth_result(self.auth_username, 'keyboard-interactive', result) + + + _handler_table = { + MSG_SERVICE_REQUEST: _parse_service_request, + MSG_SERVICE_ACCEPT: _parse_service_accept, + MSG_USERAUTH_REQUEST: _parse_userauth_request, + MSG_USERAUTH_SUCCESS: _parse_userauth_success, + MSG_USERAUTH_FAILURE: _parse_userauth_failure, + MSG_USERAUTH_BANNER: _parse_userauth_banner, + MSG_USERAUTH_INFO_REQUEST: _parse_userauth_info_request, + MSG_USERAUTH_INFO_RESPONSE: _parse_userauth_info_response, + } + diff --git a/contrib/site-packages/paramiko/ber.py b/contrib/site-packages/paramiko/ber.py new file mode 100644 index 00000000..3941581c --- /dev/null +++ b/contrib/site-packages/paramiko/ber.py @@ -0,0 +1,129 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + + +import util + + +class BERException (Exception): + pass + + +class BER(object): + """ + Robey's tiny little attempt at a BER decoder. + """ + + def __init__(self, content=''): + self.content = content + self.idx = 0 + + def __str__(self): + return self.content + + def __repr__(self): + return 'BER(\'' + repr(self.content) + '\')' + + def decode(self): + return self.decode_next() + + def decode_next(self): + if self.idx >= len(self.content): + return None + ident = ord(self.content[self.idx]) + self.idx += 1 + if (ident & 31) == 31: + # identifier > 30 + ident = 0 + while self.idx < len(self.content): + t = ord(self.content[self.idx]) + self.idx += 1 + ident = (ident << 7) | (t & 0x7f) + if not (t & 0x80): + break + if self.idx >= len(self.content): + return None + # now fetch length + size = ord(self.content[self.idx]) + self.idx += 1 + if size & 0x80: + # more complimicated... + # FIXME: theoretically should handle indefinite-length (0x80) + t = size & 0x7f + if self.idx + t > len(self.content): + return None + size = util.inflate_long(self.content[self.idx : self.idx + t], True) + self.idx += t + if self.idx + size > len(self.content): + # can't fit + return None + data = self.content[self.idx : self.idx + size] + self.idx += size + # now switch on id + if ident == 0x30: + # sequence + return self.decode_sequence(data) + elif ident == 2: + # int + return util.inflate_long(data) + else: + # 1: boolean (00 false, otherwise true) + raise BERException('Unknown ber encoding type %d (robey is lazy)' % ident) + + def decode_sequence(data): + out = [] + b = BER(data) + while True: + x = b.decode_next() + if x is None: + break + out.append(x) + return out + decode_sequence = staticmethod(decode_sequence) + + def encode_tlv(self, ident, val): + # no need to support ident > 31 here + self.content += chr(ident) + if len(val) > 0x7f: + lenstr = util.deflate_long(len(val)) + self.content += chr(0x80 + len(lenstr)) + lenstr + else: + self.content += chr(len(val)) + self.content += val + + def encode(self, x): + if type(x) is bool: + if x: + self.encode_tlv(1, '\xff') + else: + self.encode_tlv(1, '\x00') + elif (type(x) is int) or (type(x) is long): + self.encode_tlv(2, util.deflate_long(x)) + elif type(x) is str: + self.encode_tlv(4, x) + elif (type(x) is list) or (type(x) is tuple): + self.encode_tlv(0x30, self.encode_sequence(x)) + else: + raise BERException('Unknown type for encoding: %s' % repr(type(x))) + + def encode_sequence(data): + b = BER() + for item in data: + b.encode(item) + return str(b) + encode_sequence = staticmethod(encode_sequence) diff --git a/contrib/site-packages/paramiko/buffered_pipe.py b/contrib/site-packages/paramiko/buffered_pipe.py new file mode 100644 index 00000000..4ef5cf74 --- /dev/null +++ b/contrib/site-packages/paramiko/buffered_pipe.py @@ -0,0 +1,200 @@ +# Copyright (C) 2006-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +Attempt to generalize the "feeder" part of a Channel: an object which can be +read from and closed, but is reading from a buffer fed by another thread. The +read operations are blocking and can have a timeout set. +""" + +import array +import threading +import time + + +class PipeTimeout (IOError): + """ + Indicates that a timeout was reached on a read from a L{BufferedPipe}. + """ + pass + + +class BufferedPipe (object): + """ + A buffer that obeys normal read (with timeout) & close semantics for a + file or socket, but is fed data from another thread. This is used by + L{Channel}. + """ + + def __init__(self): + self._lock = threading.Lock() + self._cv = threading.Condition(self._lock) + self._event = None + self._buffer = array.array('B') + self._closed = False + + def set_event(self, event): + """ + Set an event on this buffer. When data is ready to be read (or the + buffer has been closed), the event will be set. When no data is + ready, the event will be cleared. + + @param event: the event to set/clear + @type event: Event + """ + self._event = event + if len(self._buffer) > 0: + event.set() + else: + event.clear() + + def feed(self, data): + """ + Feed new data into this pipe. This method is assumed to be called + from a separate thread, so synchronization is done. + + @param data: the data to add + @type data: str + """ + self._lock.acquire() + try: + if self._event is not None: + self._event.set() + self._buffer.fromstring(data) + self._cv.notifyAll() + finally: + self._lock.release() + + def read_ready(self): + """ + Returns true if data is buffered and ready to be read from this + feeder. A C{False} result does not mean that the feeder has closed; + it means you may need to wait before more data arrives. + + @return: C{True} if a L{read} call would immediately return at least + one byte; C{False} otherwise. + @rtype: bool + """ + self._lock.acquire() + try: + if len(self._buffer) == 0: + return False + return True + finally: + self._lock.release() + + def read(self, nbytes, timeout=None): + """ + Read data from the pipe. The return value is a string representing + the data received. The maximum amount of data to be received at once + is specified by C{nbytes}. If a string of length zero is returned, + the pipe has been closed. + + The optional C{timeout} argument can be a nonnegative float expressing + seconds, or C{None} for no timeout. If a float is given, a + C{PipeTimeout} will be raised if the timeout period value has + elapsed before any data arrives. + + @param nbytes: maximum number of bytes to read + @type nbytes: int + @param timeout: maximum seconds to wait (or C{None}, the default, to + wait forever) + @type timeout: float + @return: data + @rtype: str + + @raise PipeTimeout: if a timeout was specified and no data was ready + before that timeout + """ + out = '' + self._lock.acquire() + try: + if len(self._buffer) == 0: + if self._closed: + return out + # should we block? + if timeout == 0.0: + raise PipeTimeout() + # loop here in case we get woken up but a different thread has + # grabbed everything in the buffer. + while (len(self._buffer) == 0) and not self._closed: + then = time.time() + self._cv.wait(timeout) + if timeout is not None: + timeout -= time.time() - then + if timeout <= 0.0: + raise PipeTimeout() + + # something's in the buffer and we have the lock! + if len(self._buffer) <= nbytes: + out = self._buffer.tostring() + del self._buffer[:] + if (self._event is not None) and not self._closed: + self._event.clear() + else: + out = self._buffer[:nbytes].tostring() + del self._buffer[:nbytes] + finally: + self._lock.release() + + return out + + def empty(self): + """ + Clear out the buffer and return all data that was in it. + + @return: any data that was in the buffer prior to clearing it out + @rtype: str + """ + self._lock.acquire() + try: + out = self._buffer.tostring() + del self._buffer[:] + if (self._event is not None) and not self._closed: + self._event.clear() + return out + finally: + self._lock.release() + + def close(self): + """ + Close this pipe object. Future calls to L{read} after the buffer + has been emptied will return immediately with an empty string. + """ + self._lock.acquire() + try: + self._closed = True + self._cv.notifyAll() + if self._event is not None: + self._event.set() + finally: + self._lock.release() + + def __len__(self): + """ + Return the number of bytes buffered. + + @return: number of bytes bufferes + @rtype: int + """ + self._lock.acquire() + try: + return len(self._buffer) + finally: + self._lock.release() + diff --git a/contrib/site-packages/paramiko/channel.py b/contrib/site-packages/paramiko/channel.py new file mode 100644 index 00000000..d1e6333c --- /dev/null +++ b/contrib/site-packages/paramiko/channel.py @@ -0,0 +1,1279 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +Abstraction for an SSH2 channel. +""" + +import binascii +import sys +import time +import threading +import socket +import os + +from paramiko.common import * +from paramiko import util +from paramiko.message import Message +from paramiko.ssh_exception import SSHException +from paramiko.file import BufferedFile +from paramiko.buffered_pipe import BufferedPipe, PipeTimeout +from paramiko import pipe + + +# lower bound on the max packet size we'll accept from the remote host +MIN_PACKET_SIZE = 1024 + + +class Channel (object): + """ + A secure tunnel across an SSH L{Transport}. A Channel is meant to behave + like a socket, and has an API that should be indistinguishable from the + python socket API. + + Because SSH2 has a windowing kind of flow control, if you stop reading data + from a Channel and its buffer fills up, the server will be unable to send + you any more data until you read some of it. (This won't affect other + channels on the same transport -- all channels on a single transport are + flow-controlled independently.) Similarly, if the server isn't reading + data you send, calls to L{send} may block, unless you set a timeout. This + is exactly like a normal network socket, so it shouldn't be too surprising. + """ + + def __init__(self, chanid): + """ + Create a new channel. The channel is not associated with any + particular session or L{Transport} until the Transport attaches it. + Normally you would only call this method from the constructor of a + subclass of L{Channel}. + + @param chanid: the ID of this channel, as passed by an existing + L{Transport}. + @type chanid: int + """ + self.chanid = chanid + self.remote_chanid = 0 + self.transport = None + self.active = False + self.eof_received = 0 + self.eof_sent = 0 + self.in_buffer = BufferedPipe() + self.in_stderr_buffer = BufferedPipe() + self.timeout = None + self.closed = False + self.ultra_debug = False + self.lock = threading.Lock() + self.out_buffer_cv = threading.Condition(self.lock) + self.in_window_size = 0 + self.out_window_size = 0 + self.in_max_packet_size = 0 + self.out_max_packet_size = 0 + self.in_window_threshold = 0 + self.in_window_sofar = 0 + self.status_event = threading.Event() + self._name = str(chanid) + self.logger = util.get_logger('paramiko.transport') + self._pipe = None + self.event = threading.Event() + self.event_ready = False + self.combine_stderr = False + self.exit_status = -1 + self.origin_addr = None + + def __del__(self): + try: + self.close() + except: + pass + + def __repr__(self): + """ + Return a string representation of this object, for debugging. + + @rtype: str + """ + out = ' 0: + out += ' in-buffer=%d' % (len(self.in_buffer),) + out += ' -> ' + repr(self.transport) + out += '>' + return out + + def get_pty(self, term='vt100', width=80, height=24, width_pixels=0, + height_pixels=0): + """ + Request a pseudo-terminal from the server. This is usually used right + after creating a client channel, to ask the server to provide some + basic terminal semantics for a shell invoked with L{invoke_shell}. + It isn't necessary (or desirable) to call this method if you're going + to exectue a single command with L{exec_command}. + + @param term: the terminal type to emulate (for example, C{'vt100'}) + @type term: str + @param width: width (in characters) of the terminal screen + @type width: int + @param height: height (in characters) of the terminal screen + @type height: int + @param width_pixels: width (in pixels) of the terminal screen + @type width_pixels: int + @param height_pixels: height (in pixels) of the terminal screen + @type height_pixels: int + + @raise SSHException: if the request was rejected or the channel was + closed + """ + if self.closed or self.eof_received or self.eof_sent or not self.active: + raise SSHException('Channel is not open') + m = Message() + m.add_byte(chr(MSG_CHANNEL_REQUEST)) + m.add_int(self.remote_chanid) + m.add_string('pty-req') + m.add_boolean(True) + m.add_string(term) + m.add_int(width) + m.add_int(height) + m.add_int(width_pixels) + m.add_int(height_pixels) + m.add_string('') + self._event_pending() + self.transport._send_user_message(m) + self._wait_for_event() + + def invoke_shell(self): + """ + Request an interactive shell session on this channel. If the server + allows it, the channel will then be directly connected to the stdin, + stdout, and stderr of the shell. + + Normally you would call L{get_pty} before this, in which case the + shell will operate through the pty, and the channel will be connected + to the stdin and stdout of the pty. + + When the shell exits, the channel will be closed and can't be reused. + You must open a new channel if you wish to open another shell. + + @raise SSHException: if the request was rejected or the channel was + closed + """ + if self.closed or self.eof_received or self.eof_sent or not self.active: + raise SSHException('Channel is not open') + m = Message() + m.add_byte(chr(MSG_CHANNEL_REQUEST)) + m.add_int(self.remote_chanid) + m.add_string('shell') + m.add_boolean(1) + self._event_pending() + self.transport._send_user_message(m) + self._wait_for_event() + + def exec_command(self, command): + """ + Execute a command on the server. If the server allows it, the channel + will then be directly connected to the stdin, stdout, and stderr of + the command being executed. + + When the command finishes executing, the channel will be closed and + can't be reused. You must open a new channel if you wish to execute + another command. + + @param command: a shell command to execute. + @type command: str + + @raise SSHException: if the request was rejected or the channel was + closed + """ + if self.closed or self.eof_received or self.eof_sent or not self.active: + raise SSHException('Channel is not open') + m = Message() + m.add_byte(chr(MSG_CHANNEL_REQUEST)) + m.add_int(self.remote_chanid) + m.add_string('exec') + m.add_boolean(True) + m.add_string(command) + self._event_pending() + self.transport._send_user_message(m) + self._wait_for_event() + + def invoke_subsystem(self, subsystem): + """ + Request a subsystem on the server (for example, C{sftp}). If the + server allows it, the channel will then be directly connected to the + requested subsystem. + + When the subsystem finishes, the channel will be closed and can't be + reused. + + @param subsystem: name of the subsystem being requested. + @type subsystem: str + + @raise SSHException: if the request was rejected or the channel was + closed + """ + if self.closed or self.eof_received or self.eof_sent or not self.active: + raise SSHException('Channel is not open') + m = Message() + m.add_byte(chr(MSG_CHANNEL_REQUEST)) + m.add_int(self.remote_chanid) + m.add_string('subsystem') + m.add_boolean(True) + m.add_string(subsystem) + self._event_pending() + self.transport._send_user_message(m) + self._wait_for_event() + + def resize_pty(self, width=80, height=24, width_pixels=0, height_pixels=0): + """ + Resize the pseudo-terminal. This can be used to change the width and + height of the terminal emulation created in a previous L{get_pty} call. + + @param width: new width (in characters) of the terminal screen + @type width: int + @param height: new height (in characters) of the terminal screen + @type height: int + @param width_pixels: new width (in pixels) of the terminal screen + @type width_pixels: int + @param height_pixels: new height (in pixels) of the terminal screen + @type height_pixels: int + + @raise SSHException: if the request was rejected or the channel was + closed + """ + if self.closed or self.eof_received or self.eof_sent or not self.active: + raise SSHException('Channel is not open') + m = Message() + m.add_byte(chr(MSG_CHANNEL_REQUEST)) + m.add_int(self.remote_chanid) + m.add_string('window-change') + m.add_boolean(False) + m.add_int(width) + m.add_int(height) + m.add_int(width_pixels) + m.add_int(height_pixels) + self.transport._send_user_message(m) + + def exit_status_ready(self): + """ + Return true if the remote process has exited and returned an exit + status. You may use this to poll the process status if you don't + want to block in L{recv_exit_status}. Note that the server may not + return an exit status in some cases (like bad servers). + + @return: True if L{recv_exit_status} will return immediately + @rtype: bool + @since: 1.7.3 + """ + return self.closed or self.status_event.isSet() + + def recv_exit_status(self): + """ + Return the exit status from the process on the server. This is + mostly useful for retrieving the reults of an L{exec_command}. + If the command hasn't finished yet, this method will wait until + it does, or until the channel is closed. If no exit status is + provided by the server, -1 is returned. + + @return: the exit code of the process on the server. + @rtype: int + + @since: 1.2 + """ + self.status_event.wait() + assert self.status_event.isSet() + return self.exit_status + + def send_exit_status(self, status): + """ + Send the exit status of an executed command to the client. (This + really only makes sense in server mode.) Many clients expect to + get some sort of status code back from an executed command after + it completes. + + @param status: the exit code of the process + @type status: int + + @since: 1.2 + """ + # in many cases, the channel will not still be open here. + # that's fine. + m = Message() + m.add_byte(chr(MSG_CHANNEL_REQUEST)) + m.add_int(self.remote_chanid) + m.add_string('exit-status') + m.add_boolean(False) + m.add_int(status) + self.transport._send_user_message(m) + + def request_x11(self, screen_number=0, auth_protocol=None, auth_cookie=None, + single_connection=False, handler=None): + """ + Request an x11 session on this channel. If the server allows it, + further x11 requests can be made from the server to the client, + when an x11 application is run in a shell session. + + From RFC4254:: + + It is RECOMMENDED that the 'x11 authentication cookie' that is + sent be a fake, random cookie, and that the cookie be checked and + replaced by the real cookie when a connection request is received. + + If you omit the auth_cookie, a new secure random 128-bit value will be + generated, used, and returned. You will need to use this value to + verify incoming x11 requests and replace them with the actual local + x11 cookie (which requires some knoweldge of the x11 protocol). + + If a handler is passed in, the handler is called from another thread + whenever a new x11 connection arrives. The default handler queues up + incoming x11 connections, which may be retrieved using + L{Transport.accept}. The handler's calling signature is:: + + handler(channel: Channel, (address: str, port: int)) + + @param screen_number: the x11 screen number (0, 10, etc) + @type screen_number: int + @param auth_protocol: the name of the X11 authentication method used; + if none is given, C{"MIT-MAGIC-COOKIE-1"} is used + @type auth_protocol: str + @param auth_cookie: hexadecimal string containing the x11 auth cookie; + if none is given, a secure random 128-bit value is generated + @type auth_cookie: str + @param single_connection: if True, only a single x11 connection will be + forwarded (by default, any number of x11 connections can arrive + over this session) + @type single_connection: bool + @param handler: an optional handler to use for incoming X11 connections + @type handler: function + @return: the auth_cookie used + """ + if self.closed or self.eof_received or self.eof_sent or not self.active: + raise SSHException('Channel is not open') + if auth_protocol is None: + auth_protocol = 'MIT-MAGIC-COOKIE-1' + if auth_cookie is None: + auth_cookie = binascii.hexlify(self.transport.rng.read(16)) + + m = Message() + m.add_byte(chr(MSG_CHANNEL_REQUEST)) + m.add_int(self.remote_chanid) + m.add_string('x11-req') + m.add_boolean(True) + m.add_boolean(single_connection) + m.add_string(auth_protocol) + m.add_string(auth_cookie) + m.add_int(screen_number) + self._event_pending() + self.transport._send_user_message(m) + self._wait_for_event() + self.transport._set_x11_handler(handler) + return auth_cookie + + def request_forward_agent(self, handler): + """ + Request for a forward SSH Agent on this channel. + This is only valid for an ssh-agent from openssh !!! + + @param handler: a required handler to use for incoming SSH Agent connections + @type handler: function + + @return: if we are ok or not (at that time we always return ok) + @rtype: boolean + + @raise: SSHException in case of channel problem. + """ + if self.closed or self.eof_received or self.eof_sent or not self.active: + raise SSHException('Channel is not open') + + m = Message() + m.add_byte(chr(MSG_CHANNEL_REQUEST)) + m.add_int(self.remote_chanid) + m.add_string('auth-agent-req@openssh.com') + m.add_boolean(False) + self.transport._send_user_message(m) + self.transport._set_forward_agent_handler(handler) + return True + + def get_transport(self): + """ + Return the L{Transport} associated with this channel. + + @return: the L{Transport} that was used to create this channel. + @rtype: L{Transport} + """ + return self.transport + + def set_name(self, name): + """ + Set a name for this channel. Currently it's only used to set the name + of the channel in logfile entries. The name can be fetched with the + L{get_name} method. + + @param name: new channel name + @type name: str + """ + self._name = name + + def get_name(self): + """ + Get the name of this channel that was previously set by L{set_name}. + + @return: the name of this channel. + @rtype: str + """ + return self._name + + def get_id(self): + """ + Return the ID # for this channel. The channel ID is unique across + a L{Transport} and usually a small number. It's also the number + passed to L{ServerInterface.check_channel_request} when determining + whether to accept a channel request in server mode. + + @return: the ID of this channel. + @rtype: int + """ + return self.chanid + + def set_combine_stderr(self, combine): + """ + Set whether stderr should be combined into stdout on this channel. + The default is C{False}, but in some cases it may be convenient to + have both streams combined. + + If this is C{False}, and L{exec_command} is called (or C{invoke_shell} + with no pty), output to stderr will not show up through the L{recv} + and L{recv_ready} calls. You will have to use L{recv_stderr} and + L{recv_stderr_ready} to get stderr output. + + If this is C{True}, data will never show up via L{recv_stderr} or + L{recv_stderr_ready}. + + @param combine: C{True} if stderr output should be combined into + stdout on this channel. + @type combine: bool + @return: previous setting. + @rtype: bool + + @since: 1.1 + """ + data = '' + self.lock.acquire() + try: + old = self.combine_stderr + self.combine_stderr = combine + if combine and not old: + # copy old stderr buffer into primary buffer + data = self.in_stderr_buffer.empty() + finally: + self.lock.release() + if len(data) > 0: + self._feed(data) + return old + + + ### socket API + + + def settimeout(self, timeout): + """ + Set a timeout on blocking read/write operations. The C{timeout} + argument can be a nonnegative float expressing seconds, or C{None}. If + a float is given, subsequent channel read/write operations will raise + a timeout exception if the timeout period value has elapsed before the + operation has completed. Setting a timeout of C{None} disables + timeouts on socket operations. + + C{chan.settimeout(0.0)} is equivalent to C{chan.setblocking(0)}; + C{chan.settimeout(None)} is equivalent to C{chan.setblocking(1)}. + + @param timeout: seconds to wait for a pending read/write operation + before raising C{socket.timeout}, or C{None} for no timeout. + @type timeout: float + """ + self.timeout = timeout + + def gettimeout(self): + """ + Returns the timeout in seconds (as a float) associated with socket + operations, or C{None} if no timeout is set. This reflects the last + call to L{setblocking} or L{settimeout}. + + @return: timeout in seconds, or C{None}. + @rtype: float + """ + return self.timeout + + def setblocking(self, blocking): + """ + Set blocking or non-blocking mode of the channel: if C{blocking} is 0, + the channel is set to non-blocking mode; otherwise it's set to blocking + mode. Initially all channels are in blocking mode. + + In non-blocking mode, if a L{recv} call doesn't find any data, or if a + L{send} call can't immediately dispose of the data, an error exception + is raised. In blocking mode, the calls block until they can proceed. An + EOF condition is considered "immediate data" for L{recv}, so if the + channel is closed in the read direction, it will never block. + + C{chan.setblocking(0)} is equivalent to C{chan.settimeout(0)}; + C{chan.setblocking(1)} is equivalent to C{chan.settimeout(None)}. + + @param blocking: 0 to set non-blocking mode; non-0 to set blocking + mode. + @type blocking: int + """ + if blocking: + self.settimeout(None) + else: + self.settimeout(0.0) + + def getpeername(self): + """ + Return the address of the remote side of this Channel, if possible. + This is just a wrapper around C{'getpeername'} on the Transport, used + to provide enough of a socket-like interface to allow asyncore to work. + (asyncore likes to call C{'getpeername'}.) + + @return: the address if the remote host, if known + @rtype: tuple(str, int) + """ + return self.transport.getpeername() + + def close(self): + """ + Close the channel. All future read/write operations on the channel + will fail. The remote end will receive no more data (after queued data + is flushed). Channels are automatically closed when their L{Transport} + is closed or when they are garbage collected. + """ + self.lock.acquire() + try: + # only close the pipe when the user explicitly closes the channel. + # otherwise they will get unpleasant surprises. (and do it before + # checking self.closed, since the remote host may have already + # closed the connection.) + if self._pipe is not None: + self._pipe.close() + self._pipe = None + + if not self.active or self.closed: + return + msgs = self._close_internal() + finally: + self.lock.release() + for m in msgs: + if m is not None: + self.transport._send_user_message(m) + + def recv_ready(self): + """ + Returns true if data is buffered and ready to be read from this + channel. A C{False} result does not mean that the channel has closed; + it means you may need to wait before more data arrives. + + @return: C{True} if a L{recv} call on this channel would immediately + return at least one byte; C{False} otherwise. + @rtype: boolean + """ + return self.in_buffer.read_ready() + + def recv(self, nbytes): + """ + Receive data from the channel. The return value is a string + representing the data received. The maximum amount of data to be + received at once is specified by C{nbytes}. If a string of length zero + is returned, the channel stream has closed. + + @param nbytes: maximum number of bytes to read. + @type nbytes: int + @return: data. + @rtype: str + + @raise socket.timeout: if no data is ready before the timeout set by + L{settimeout}. + """ + try: + out = self.in_buffer.read(nbytes, self.timeout) + except PipeTimeout, e: + raise socket.timeout() + + ack = self._check_add_window(len(out)) + # no need to hold the channel lock when sending this + if ack > 0: + m = Message() + m.add_byte(chr(MSG_CHANNEL_WINDOW_ADJUST)) + m.add_int(self.remote_chanid) + m.add_int(ack) + self.transport._send_user_message(m) + + return out + + def recv_stderr_ready(self): + """ + Returns true if data is buffered and ready to be read from this + channel's stderr stream. Only channels using L{exec_command} or + L{invoke_shell} without a pty will ever have data on the stderr + stream. + + @return: C{True} if a L{recv_stderr} call on this channel would + immediately return at least one byte; C{False} otherwise. + @rtype: boolean + + @since: 1.1 + """ + return self.in_stderr_buffer.read_ready() + + def recv_stderr(self, nbytes): + """ + Receive data from the channel's stderr stream. Only channels using + L{exec_command} or L{invoke_shell} without a pty will ever have data + on the stderr stream. The return value is a string representing the + data received. The maximum amount of data to be received at once is + specified by C{nbytes}. If a string of length zero is returned, the + channel stream has closed. + + @param nbytes: maximum number of bytes to read. + @type nbytes: int + @return: data. + @rtype: str + + @raise socket.timeout: if no data is ready before the timeout set by + L{settimeout}. + + @since: 1.1 + """ + try: + out = self.in_stderr_buffer.read(nbytes, self.timeout) + except PipeTimeout, e: + raise socket.timeout() + + ack = self._check_add_window(len(out)) + # no need to hold the channel lock when sending this + if ack > 0: + m = Message() + m.add_byte(chr(MSG_CHANNEL_WINDOW_ADJUST)) + m.add_int(self.remote_chanid) + m.add_int(ack) + self.transport._send_user_message(m) + + return out + + def send_ready(self): + """ + Returns true if data can be written to this channel without blocking. + This means the channel is either closed (so any write attempt would + return immediately) or there is at least one byte of space in the + outbound buffer. If there is at least one byte of space in the + outbound buffer, a L{send} call will succeed immediately and return + the number of bytes actually written. + + @return: C{True} if a L{send} call on this channel would immediately + succeed or fail + @rtype: boolean + """ + self.lock.acquire() + try: + if self.closed or self.eof_sent: + return True + return self.out_window_size > 0 + finally: + self.lock.release() + + def send(self, s): + """ + Send data to the channel. Returns the number of bytes sent, or 0 if + the channel stream is closed. Applications are responsible for + checking that all data has been sent: if only some of the data was + transmitted, the application needs to attempt delivery of the remaining + data. + + @param s: data to send + @type s: str + @return: number of bytes actually sent + @rtype: int + + @raise socket.timeout: if no data could be sent before the timeout set + by L{settimeout}. + """ + size = len(s) + self.lock.acquire() + try: + size = self._wait_for_send_window(size) + if size == 0: + # eof or similar + return 0 + m = Message() + m.add_byte(chr(MSG_CHANNEL_DATA)) + m.add_int(self.remote_chanid) + m.add_string(s[:size]) + finally: + self.lock.release() + # Note: We release self.lock before calling _send_user_message. + # Otherwise, we can deadlock during re-keying. + self.transport._send_user_message(m) + return size + + def send_stderr(self, s): + """ + Send data to the channel on the "stderr" stream. This is normally + only used by servers to send output from shell commands -- clients + won't use this. Returns the number of bytes sent, or 0 if the channel + stream is closed. Applications are responsible for checking that all + data has been sent: if only some of the data was transmitted, the + application needs to attempt delivery of the remaining data. + + @param s: data to send. + @type s: str + @return: number of bytes actually sent. + @rtype: int + + @raise socket.timeout: if no data could be sent before the timeout set + by L{settimeout}. + + @since: 1.1 + """ + size = len(s) + self.lock.acquire() + try: + size = self._wait_for_send_window(size) + if size == 0: + # eof or similar + return 0 + m = Message() + m.add_byte(chr(MSG_CHANNEL_EXTENDED_DATA)) + m.add_int(self.remote_chanid) + m.add_int(1) + m.add_string(s[:size]) + finally: + self.lock.release() + # Note: We release self.lock before calling _send_user_message. + # Otherwise, we can deadlock during re-keying. + self.transport._send_user_message(m) + return size + + def sendall(self, s): + """ + Send data to the channel, without allowing partial results. Unlike + L{send}, this method continues to send data from the given string until + either all data has been sent or an error occurs. Nothing is returned. + + @param s: data to send. + @type s: str + + @raise socket.timeout: if sending stalled for longer than the timeout + set by L{settimeout}. + @raise socket.error: if an error occured before the entire string was + sent. + + @note: If the channel is closed while only part of the data hase been + sent, there is no way to determine how much data (if any) was sent. + This is irritating, but identically follows python's API. + """ + while s: + if self.closed: + # this doesn't seem useful, but it is the documented behavior of Socket + raise socket.error('Socket is closed') + sent = self.send(s) + s = s[sent:] + return None + + def sendall_stderr(self, s): + """ + Send data to the channel's "stderr" stream, without allowing partial + results. Unlike L{send_stderr}, this method continues to send data + from the given string until all data has been sent or an error occurs. + Nothing is returned. + + @param s: data to send to the client as "stderr" output. + @type s: str + + @raise socket.timeout: if sending stalled for longer than the timeout + set by L{settimeout}. + @raise socket.error: if an error occured before the entire string was + sent. + + @since: 1.1 + """ + while s: + if self.closed: + raise socket.error('Socket is closed') + sent = self.send_stderr(s) + s = s[sent:] + return None + + def makefile(self, *params): + """ + Return a file-like object associated with this channel. The optional + C{mode} and C{bufsize} arguments are interpreted the same way as by + the built-in C{file()} function in python. + + @return: object which can be used for python file I/O. + @rtype: L{ChannelFile} + """ + return ChannelFile(*([self] + list(params))) + + def makefile_stderr(self, *params): + """ + Return a file-like object associated with this channel's stderr + stream. Only channels using L{exec_command} or L{invoke_shell} + without a pty will ever have data on the stderr stream. + + The optional C{mode} and C{bufsize} arguments are interpreted the + same way as by the built-in C{file()} function in python. For a + client, it only makes sense to open this file for reading. For a + server, it only makes sense to open this file for writing. + + @return: object which can be used for python file I/O. + @rtype: L{ChannelFile} + + @since: 1.1 + """ + return ChannelStderrFile(*([self] + list(params))) + + def fileno(self): + """ + Returns an OS-level file descriptor which can be used for polling, but + but I{not} for reading or writing. This is primaily to allow python's + C{select} module to work. + + The first time C{fileno} is called on a channel, a pipe is created to + simulate real OS-level file descriptor (FD) behavior. Because of this, + two OS-level FDs are created, which will use up FDs faster than normal. + (You won't notice this effect unless you have hundreds of channels + open at the same time.) + + @return: an OS-level file descriptor + @rtype: int + + @warning: This method causes channel reads to be slightly less + efficient. + """ + self.lock.acquire() + try: + if self._pipe is not None: + return self._pipe.fileno() + # create the pipe and feed in any existing data + self._pipe = pipe.make_pipe() + p1, p2 = pipe.make_or_pipe(self._pipe) + self.in_buffer.set_event(p1) + self.in_stderr_buffer.set_event(p2) + return self._pipe.fileno() + finally: + self.lock.release() + + def shutdown(self, how): + """ + Shut down one or both halves of the connection. If C{how} is 0, + further receives are disallowed. If C{how} is 1, further sends + are disallowed. If C{how} is 2, further sends and receives are + disallowed. This closes the stream in one or both directions. + + @param how: 0 (stop receiving), 1 (stop sending), or 2 (stop + receiving and sending). + @type how: int + """ + if (how == 0) or (how == 2): + # feign "read" shutdown + self.eof_received = 1 + if (how == 1) or (how == 2): + self.lock.acquire() + try: + m = self._send_eof() + finally: + self.lock.release() + if m is not None: + self.transport._send_user_message(m) + + def shutdown_read(self): + """ + Shutdown the receiving side of this socket, closing the stream in + the incoming direction. After this call, future reads on this + channel will fail instantly. This is a convenience method, equivalent + to C{shutdown(0)}, for people who don't make it a habit to + memorize unix constants from the 1970s. + + @since: 1.2 + """ + self.shutdown(0) + + def shutdown_write(self): + """ + Shutdown the sending side of this socket, closing the stream in + the outgoing direction. After this call, future writes on this + channel will fail instantly. This is a convenience method, equivalent + to C{shutdown(1)}, for people who don't make it a habit to + memorize unix constants from the 1970s. + + @since: 1.2 + """ + self.shutdown(1) + + + ### calls from Transport + + + def _set_transport(self, transport): + self.transport = transport + self.logger = util.get_logger(self.transport.get_log_channel()) + + def _set_window(self, window_size, max_packet_size): + self.in_window_size = window_size + self.in_max_packet_size = max_packet_size + # threshold of bytes we receive before we bother to send a window update + self.in_window_threshold = window_size // 10 + self.in_window_sofar = 0 + self._log(DEBUG, 'Max packet in: %d bytes' % max_packet_size) + + def _set_remote_channel(self, chanid, window_size, max_packet_size): + self.remote_chanid = chanid + self.out_window_size = window_size + self.out_max_packet_size = max(max_packet_size, MIN_PACKET_SIZE) + self.active = 1 + self._log(DEBUG, 'Max packet out: %d bytes' % max_packet_size) + + def _request_success(self, m): + self._log(DEBUG, 'Sesch channel %d request ok' % self.chanid) + self.event_ready = True + self.event.set() + return + + def _request_failed(self, m): + self.lock.acquire() + try: + msgs = self._close_internal() + finally: + self.lock.release() + for m in msgs: + if m is not None: + self.transport._send_user_message(m) + + def _feed(self, m): + if type(m) is str: + # passed from _feed_extended + s = m + else: + s = m.get_string() + self.in_buffer.feed(s) + + def _feed_extended(self, m): + code = m.get_int() + s = m.get_string() + if code != 1: + self._log(ERROR, 'unknown extended_data type %d; discarding' % code) + return + if self.combine_stderr: + self._feed(s) + else: + self.in_stderr_buffer.feed(s) + + def _window_adjust(self, m): + nbytes = m.get_int() + self.lock.acquire() + try: + if self.ultra_debug: + self._log(DEBUG, 'window up %d' % nbytes) + self.out_window_size += nbytes + self.out_buffer_cv.notifyAll() + finally: + self.lock.release() + + def _handle_request(self, m): + key = m.get_string() + want_reply = m.get_boolean() + server = self.transport.server_object + ok = False + if key == 'exit-status': + self.exit_status = m.get_int() + self.status_event.set() + ok = True + elif key == 'xon-xoff': + # ignore + ok = True + elif key == 'pty-req': + term = m.get_string() + width = m.get_int() + height = m.get_int() + pixelwidth = m.get_int() + pixelheight = m.get_int() + modes = m.get_string() + if server is None: + ok = False + else: + ok = server.check_channel_pty_request(self, term, width, height, pixelwidth, + pixelheight, modes) + elif key == 'shell': + if server is None: + ok = False + else: + ok = server.check_channel_shell_request(self) + elif key == 'env': + name = m.get_string() + value = m.get_string() + if server is None: + ok = False + else: + ok = server.check_channel_env_request(self, name, value) + elif key == 'exec': + cmd = m.get_string() + if server is None: + ok = False + else: + ok = server.check_channel_exec_request(self, cmd) + elif key == 'subsystem': + name = m.get_string() + if server is None: + ok = False + else: + ok = server.check_channel_subsystem_request(self, name) + elif key == 'window-change': + width = m.get_int() + height = m.get_int() + pixelwidth = m.get_int() + pixelheight = m.get_int() + if server is None: + ok = False + else: + ok = server.check_channel_window_change_request(self, width, height, pixelwidth, + pixelheight) + elif key == 'x11-req': + single_connection = m.get_boolean() + auth_proto = m.get_string() + auth_cookie = m.get_string() + screen_number = m.get_int() + if server is None: + ok = False + else: + ok = server.check_channel_x11_request(self, single_connection, + auth_proto, auth_cookie, screen_number) + elif key == 'auth-agent-req@openssh.com': + if server is None: + ok = False + else: + ok = server.check_channel_forward_agent_request(self) + else: + self._log(DEBUG, 'Unhandled channel request "%s"' % key) + ok = False + if want_reply: + m = Message() + if ok: + m.add_byte(chr(MSG_CHANNEL_SUCCESS)) + else: + m.add_byte(chr(MSG_CHANNEL_FAILURE)) + m.add_int(self.remote_chanid) + self.transport._send_user_message(m) + + def _handle_eof(self, m): + self.lock.acquire() + try: + if not self.eof_received: + self.eof_received = True + self.in_buffer.close() + self.in_stderr_buffer.close() + if self._pipe is not None: + self._pipe.set_forever() + finally: + self.lock.release() + self._log(DEBUG, 'EOF received (%s)', self._name) + + def _handle_close(self, m): + self.lock.acquire() + try: + msgs = self._close_internal() + self.transport._unlink_channel(self.chanid) + finally: + self.lock.release() + for m in msgs: + if m is not None: + self.transport._send_user_message(m) + + + ### internals... + + + def _log(self, level, msg, *args): + self.logger.log(level, "[chan " + self._name + "] " + msg, *args) + + def _event_pending(self): + self.event.clear() + self.event_ready = False + + def _wait_for_event(self): + self.event.wait() + assert self.event.isSet() + if self.event_ready: + return + e = self.transport.get_exception() + if e is None: + e = SSHException('Channel closed.') + raise e + + def _set_closed(self): + # you are holding the lock. + self.closed = True + self.in_buffer.close() + self.in_stderr_buffer.close() + self.out_buffer_cv.notifyAll() + # Notify any waiters that we are closed + self.event.set() + self.status_event.set() + if self._pipe is not None: + self._pipe.set_forever() + + def _send_eof(self): + # you are holding the lock. + if self.eof_sent: + return None + m = Message() + m.add_byte(chr(MSG_CHANNEL_EOF)) + m.add_int(self.remote_chanid) + self.eof_sent = True + self._log(DEBUG, 'EOF sent (%s)', self._name) + return m + + def _close_internal(self): + # you are holding the lock. + if not self.active or self.closed: + return None, None + m1 = self._send_eof() + m2 = Message() + m2.add_byte(chr(MSG_CHANNEL_CLOSE)) + m2.add_int(self.remote_chanid) + self._set_closed() + # can't unlink from the Transport yet -- the remote side may still + # try to send meta-data (exit-status, etc) + return m1, m2 + + def _unlink(self): + # server connection could die before we become active: still signal the close! + if self.closed: + return + self.lock.acquire() + try: + self._set_closed() + self.transport._unlink_channel(self.chanid) + finally: + self.lock.release() + + def _check_add_window(self, n): + self.lock.acquire() + try: + if self.closed or self.eof_received or not self.active: + return 0 + if self.ultra_debug: + self._log(DEBUG, 'addwindow %d' % n) + self.in_window_sofar += n + if self.in_window_sofar <= self.in_window_threshold: + return 0 + if self.ultra_debug: + self._log(DEBUG, 'addwindow send %d' % self.in_window_sofar) + out = self.in_window_sofar + self.in_window_sofar = 0 + return out + finally: + self.lock.release() + + def _wait_for_send_window(self, size): + """ + (You are already holding the lock.) + Wait for the send window to open up, and allocate up to C{size} bytes + for transmission. If no space opens up before the timeout, a timeout + exception is raised. Returns the number of bytes available to send + (may be less than requested). + """ + # you are already holding the lock + if self.closed or self.eof_sent: + return 0 + if self.out_window_size == 0: + # should we block? + if self.timeout == 0.0: + raise socket.timeout() + # loop here in case we get woken up but a different thread has filled the buffer + timeout = self.timeout + while self.out_window_size == 0: + if self.closed or self.eof_sent: + return 0 + then = time.time() + self.out_buffer_cv.wait(timeout) + if timeout != None: + timeout -= time.time() - then + if timeout <= 0.0: + raise socket.timeout() + # we have some window to squeeze into + if self.closed or self.eof_sent: + return 0 + if self.out_window_size < size: + size = self.out_window_size + if self.out_max_packet_size - 64 < size: + size = self.out_max_packet_size - 64 + self.out_window_size -= size + if self.ultra_debug: + self._log(DEBUG, 'window down to %d' % self.out_window_size) + return size + + +class ChannelFile (BufferedFile): + """ + A file-like wrapper around L{Channel}. A ChannelFile is created by calling + L{Channel.makefile}. + + @bug: To correctly emulate the file object created from a socket's + C{makefile} method, a L{Channel} and its C{ChannelFile} should be able + to be closed or garbage-collected independently. Currently, closing + the C{ChannelFile} does nothing but flush the buffer. + """ + + def __init__(self, channel, mode = 'r', bufsize = -1): + self.channel = channel + BufferedFile.__init__(self) + self._set_mode(mode, bufsize) + + def __repr__(self): + """ + Returns a string representation of this object, for debugging. + + @rtype: str + """ + return '' + + def _read(self, size): + return self.channel.recv(size) + + def _write(self, data): + self.channel.sendall(data) + return len(data) + + +class ChannelStderrFile (ChannelFile): + def __init__(self, channel, mode = 'r', bufsize = -1): + ChannelFile.__init__(self, channel, mode, bufsize) + + def _read(self, size): + return self.channel.recv_stderr(size) + + def _write(self, data): + self.channel.sendall_stderr(data) + return len(data) + + +# vim: set shiftwidth=4 expandtab : diff --git a/contrib/site-packages/paramiko/client.py b/contrib/site-packages/paramiko/client.py new file mode 100644 index 00000000..c5a2d1ac --- /dev/null +++ b/contrib/site-packages/paramiko/client.py @@ -0,0 +1,538 @@ +# Copyright (C) 2006-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +L{SSHClient}. +""" + +from binascii import hexlify +import getpass +import os +import socket +import warnings + +from paramiko.agent import Agent +from paramiko.common import * +from paramiko.config import SSH_PORT +from paramiko.dsskey import DSSKey +from paramiko.hostkeys import HostKeys +from paramiko.resource import ResourceManager +from paramiko.rsakey import RSAKey +from paramiko.ssh_exception import SSHException, BadHostKeyException +from paramiko.transport import Transport +from paramiko.util import retry_on_signal + + +class MissingHostKeyPolicy (object): + """ + Interface for defining the policy that L{SSHClient} should use when the + SSH server's hostname is not in either the system host keys or the + application's keys. Pre-made classes implement policies for automatically + adding the key to the application's L{HostKeys} object (L{AutoAddPolicy}), + and for automatically rejecting the key (L{RejectPolicy}). + + This function may be used to ask the user to verify the key, for example. + """ + + def missing_host_key(self, client, hostname, key): + """ + Called when an L{SSHClient} receives a server key for a server that + isn't in either the system or local L{HostKeys} object. To accept + the key, simply return. To reject, raised an exception (which will + be passed to the calling application). + """ + pass + + +class AutoAddPolicy (MissingHostKeyPolicy): + """ + Policy for automatically adding the hostname and new host key to the + local L{HostKeys} object, and saving it. This is used by L{SSHClient}. + """ + + def missing_host_key(self, client, hostname, key): + client._host_keys.add(hostname, key.get_name(), key) + if client._host_keys_filename is not None: + client.save_host_keys(client._host_keys_filename) + client._log(DEBUG, 'Adding %s host key for %s: %s' % + (key.get_name(), hostname, hexlify(key.get_fingerprint()))) + + +class RejectPolicy (MissingHostKeyPolicy): + """ + Policy for automatically rejecting the unknown hostname & key. This is + used by L{SSHClient}. + """ + + def missing_host_key(self, client, hostname, key): + client._log(DEBUG, 'Rejecting %s host key for %s: %s' % + (key.get_name(), hostname, hexlify(key.get_fingerprint()))) + raise SSHException('Server %r not found in known_hosts' % hostname) + + +class WarningPolicy (MissingHostKeyPolicy): + """ + Policy for logging a python-style warning for an unknown host key, but + accepting it. This is used by L{SSHClient}. + """ + def missing_host_key(self, client, hostname, key): + warnings.warn('Unknown %s host key for %s: %s' % + (key.get_name(), hostname, hexlify(key.get_fingerprint()))) + + +class SSHClient (object): + """ + A high-level representation of a session with an SSH server. This class + wraps L{Transport}, L{Channel}, and L{SFTPClient} to take care of most + aspects of authenticating and opening channels. A typical use case is:: + + client = SSHClient() + client.load_system_host_keys() + client.connect('ssh.example.com') + stdin, stdout, stderr = client.exec_command('ls -l') + + You may pass in explicit overrides for authentication and server host key + checking. The default mechanism is to try to use local key files or an + SSH agent (if one is running). + + @since: 1.6 + """ + + def __init__(self): + """ + Create a new SSHClient. + """ + self._system_host_keys = HostKeys() + self._host_keys = HostKeys() + self._host_keys_filename = None + self._log_channel = None + self._policy = RejectPolicy() + self._transport = None + self._agent = None + + def load_system_host_keys(self, filename=None): + """ + Load host keys from a system (read-only) file. Host keys read with + this method will not be saved back by L{save_host_keys}. + + This method can be called multiple times. Each new set of host keys + will be merged with the existing set (new replacing old if there are + conflicts). + + If C{filename} is left as C{None}, an attempt will be made to read + keys from the user's local "known hosts" file, as used by OpenSSH, + and no exception will be raised if the file can't be read. This is + probably only useful on posix. + + @param filename: the filename to read, or C{None} + @type filename: str + + @raise IOError: if a filename was provided and the file could not be + read + """ + if filename is None: + # try the user's .ssh key file, and mask exceptions + filename = os.path.expanduser('~/.ssh/known_hosts') + try: + self._system_host_keys.load(filename) + except IOError: + pass + return + self._system_host_keys.load(filename) + + def load_host_keys(self, filename): + """ + Load host keys from a local host-key file. Host keys read with this + method will be checked I{after} keys loaded via L{load_system_host_keys}, + but will be saved back by L{save_host_keys} (so they can be modified). + The missing host key policy L{AutoAddPolicy} adds keys to this set and + saves them, when connecting to a previously-unknown server. + + This method can be called multiple times. Each new set of host keys + will be merged with the existing set (new replacing old if there are + conflicts). When automatically saving, the last hostname is used. + + @param filename: the filename to read + @type filename: str + + @raise IOError: if the filename could not be read + """ + self._host_keys_filename = filename + self._host_keys.load(filename) + + def save_host_keys(self, filename): + """ + Save the host keys back to a file. Only the host keys loaded with + L{load_host_keys} (plus any added directly) will be saved -- not any + host keys loaded with L{load_system_host_keys}. + + @param filename: the filename to save to + @type filename: str + + @raise IOError: if the file could not be written + """ + + # update local host keys from file (in case other SSH clients + # have written to the known_hosts file meanwhile. + if self.known_hosts is not None: + self.load_host_keys(self.known_hosts) + + f = open(filename, 'w') + for hostname, keys in self._host_keys.iteritems(): + for keytype, key in keys.iteritems(): + f.write('%s %s %s\n' % (hostname, keytype, key.get_base64())) + f.close() + + def get_host_keys(self): + """ + Get the local L{HostKeys} object. This can be used to examine the + local host keys or change them. + + @return: the local host keys + @rtype: L{HostKeys} + """ + return self._host_keys + + def set_log_channel(self, name): + """ + Set the channel for logging. The default is C{"paramiko.transport"} + but it can be set to anything you want. + + @param name: new channel name for logging + @type name: str + """ + self._log_channel = name + + def set_missing_host_key_policy(self, policy): + """ + Set the policy to use when connecting to a server that doesn't have a + host key in either the system or local L{HostKeys} objects. The + default policy is to reject all unknown servers (using L{RejectPolicy}). + You may substitute L{AutoAddPolicy} or write your own policy class. + + @param policy: the policy to use when receiving a host key from a + previously-unknown server + @type policy: L{MissingHostKeyPolicy} + """ + self._policy = policy + + def connect(self, hostname, port=SSH_PORT, username=None, password=None, pkey=None, + key_filename=None, timeout=None, allow_agent=True, look_for_keys=True, + compress=False, sock=None): + """ + Connect to an SSH server and authenticate to it. The server's host key + is checked against the system host keys (see L{load_system_host_keys}) + and any local host keys (L{load_host_keys}). If the server's hostname + is not found in either set of host keys, the missing host key policy + is used (see L{set_missing_host_key_policy}). The default policy is + to reject the key and raise an L{SSHException}. + + Authentication is attempted in the following order of priority: + + - The C{pkey} or C{key_filename} passed in (if any) + - Any key we can find through an SSH agent + - Any "id_rsa" or "id_dsa" key discoverable in C{~/.ssh/} + - Plain username/password auth, if a password was given + + If a private key requires a password to unlock it, and a password is + passed in, that password will be used to attempt to unlock the key. + + @param hostname: the server to connect to + @type hostname: str + @param port: the server port to connect to + @type port: int + @param username: the username to authenticate as (defaults to the + current local username) + @type username: str + @param password: a password to use for authentication or for unlocking + a private key + @type password: str + @param pkey: an optional private key to use for authentication + @type pkey: L{PKey} + @param key_filename: the filename, or list of filenames, of optional + private key(s) to try for authentication + @type key_filename: str or list(str) + @param timeout: an optional timeout (in seconds) for the TCP connect + @type timeout: float + @param allow_agent: set to False to disable connecting to the SSH agent + @type allow_agent: bool + @param look_for_keys: set to False to disable searching for discoverable + private key files in C{~/.ssh/} + @type look_for_keys: bool + @param compress: set to True to turn on compression + @type compress: bool + @param sock: an open socket or socket-like object (such as a + L{Channel}) to use for communication to the target host + @type sock: socket + + @raise BadHostKeyException: if the server's host key could not be + verified + @raise AuthenticationException: if authentication failed + @raise SSHException: if there was any other error connecting or + establishing an SSH session + @raise socket.error: if a socket error occurred while connecting + """ + if not sock: + for (family, socktype, proto, canonname, sockaddr) in socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM): + if socktype == socket.SOCK_STREAM: + af = family + addr = sockaddr + break + else: + # some OS like AIX don't indicate SOCK_STREAM support, so just guess. :( + af, _, _, _, addr = socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM) + sock = socket.socket(af, socket.SOCK_STREAM) + if timeout is not None: + try: + sock.settimeout(timeout) + except: + pass + retry_on_signal(lambda: sock.connect(addr)) + + t = self._transport = Transport(sock) + t.use_compression(compress=compress) + if self._log_channel is not None: + t.set_log_channel(self._log_channel) + t.start_client() + ResourceManager.register(self, t) + + server_key = t.get_remote_server_key() + keytype = server_key.get_name() + + if port == SSH_PORT: + server_hostkey_name = hostname + else: + server_hostkey_name = "[%s]:%d" % (hostname, port) + our_server_key = self._system_host_keys.get(server_hostkey_name, {}).get(keytype, None) + if our_server_key is None: + our_server_key = self._host_keys.get(server_hostkey_name, {}).get(keytype, None) + if our_server_key is None: + # will raise exception if the key is rejected; let that fall out + self._policy.missing_host_key(self, server_hostkey_name, server_key) + # if the callback returns, assume the key is ok + our_server_key = server_key + + if server_key != our_server_key: + raise BadHostKeyException(hostname, server_key, our_server_key) + + if username is None: + username = getpass.getuser() + + if key_filename is None: + key_filenames = [] + elif isinstance(key_filename, (str, unicode)): + key_filenames = [ key_filename ] + else: + key_filenames = key_filename + self._auth(username, password, pkey, key_filenames, allow_agent, look_for_keys) + + def close(self): + """ + Close this SSHClient and its underlying L{Transport}. + """ + if self._transport is None: + return + self._transport.close() + self._transport = None + + if self._agent != None: + self._agent.close() + self._agent = None + + def exec_command(self, command, bufsize=-1, timeout=None, get_pty=False): + """ + Execute a command on the SSH server. A new L{Channel} is opened and + the requested command is executed. The command's input and output + streams are returned as python C{file}-like objects representing + stdin, stdout, and stderr. + + @param command: the command to execute + @type command: str + @param bufsize: interpreted the same way as by the built-in C{file()} function in python + @type bufsize: int + @param timeout: set command's channel timeout. See L{Channel.settimeout}.settimeout + @type timeout: int + @return: the stdin, stdout, and stderr of the executing command + @rtype: tuple(L{ChannelFile}, L{ChannelFile}, L{ChannelFile}) + + @raise SSHException: if the server fails to execute the command + """ + chan = self._transport.open_session() + if(get_pty): + chan.get_pty() + chan.settimeout(timeout) + chan.exec_command(command) + stdin = chan.makefile('wb', bufsize) + stdout = chan.makefile('rb', bufsize) + stderr = chan.makefile_stderr('rb', bufsize) + return stdin, stdout, stderr + + def invoke_shell(self, term='vt100', width=80, height=24, width_pixels=0, + height_pixels=0): + """ + Start an interactive shell session on the SSH server. A new L{Channel} + is opened and connected to a pseudo-terminal using the requested + terminal type and size. + + @param term: the terminal type to emulate (for example, C{"vt100"}) + @type term: str + @param width: the width (in characters) of the terminal window + @type width: int + @param height: the height (in characters) of the terminal window + @type height: int + @param width_pixels: the width (in pixels) of the terminal window + @type width_pixels: int + @param height_pixels: the height (in pixels) of the terminal window + @type height_pixels: int + @return: a new channel connected to the remote shell + @rtype: L{Channel} + + @raise SSHException: if the server fails to invoke a shell + """ + chan = self._transport.open_session() + chan.get_pty(term, width, height, width_pixels, height_pixels) + chan.invoke_shell() + return chan + + def open_sftp(self): + """ + Open an SFTP session on the SSH server. + + @return: a new SFTP session object + @rtype: L{SFTPClient} + """ + return self._transport.open_sftp_client() + + def get_transport(self): + """ + Return the underlying L{Transport} object for this SSH connection. + This can be used to perform lower-level tasks, like opening specific + kinds of channels. + + @return: the Transport for this connection + @rtype: L{Transport} + """ + return self._transport + + def _auth(self, username, password, pkey, key_filenames, allow_agent, look_for_keys): + """ + Try, in order: + + - The key passed in, if one was passed in. + - Any key we can find through an SSH agent (if allowed). + - Any "id_rsa" or "id_dsa" key discoverable in ~/.ssh/ (if allowed). + - Plain username/password auth, if a password was given. + + (The password might be needed to unlock a private key, or for + two-factor authentication [for which it is required].) + """ + saved_exception = None + two_factor = False + allowed_types = [] + + if pkey is not None: + try: + self._log(DEBUG, 'Trying SSH key %s' % hexlify(pkey.get_fingerprint())) + allowed_types = self._transport.auth_publickey(username, pkey) + two_factor = (allowed_types == ['password']) + if not two_factor: + return + except SSHException, e: + saved_exception = e + + if not two_factor: + for key_filename in key_filenames: + for pkey_class in (RSAKey, DSSKey): + try: + key = pkey_class.from_private_key_file(key_filename, password) + self._log(DEBUG, 'Trying key %s from %s' % (hexlify(key.get_fingerprint()), key_filename)) + self._transport.auth_publickey(username, key) + two_factor = (allowed_types == ['password']) + if not two_factor: + return + break + except SSHException, e: + saved_exception = e + + if not two_factor and allow_agent: + if self._agent == None: + self._agent = Agent() + + for key in self._agent.get_keys(): + try: + self._log(DEBUG, 'Trying SSH agent key %s' % hexlify(key.get_fingerprint())) + # for 2-factor auth a successfully auth'd key will result in ['password'] + allowed_types = self._transport.auth_publickey(username, key) + two_factor = (allowed_types == ['password']) + if not two_factor: + return + break + except SSHException, e: + saved_exception = e + + if not two_factor: + keyfiles = [] + rsa_key = os.path.expanduser('~/.ssh/id_rsa') + dsa_key = os.path.expanduser('~/.ssh/id_dsa') + if os.path.isfile(rsa_key): + keyfiles.append((RSAKey, rsa_key)) + if os.path.isfile(dsa_key): + keyfiles.append((DSSKey, dsa_key)) + # look in ~/ssh/ for windows users: + rsa_key = os.path.expanduser('~/ssh/id_rsa') + dsa_key = os.path.expanduser('~/ssh/id_dsa') + if os.path.isfile(rsa_key): + keyfiles.append((RSAKey, rsa_key)) + if os.path.isfile(dsa_key): + keyfiles.append((DSSKey, dsa_key)) + + if not look_for_keys: + keyfiles = [] + + for pkey_class, filename in keyfiles: + try: + key = pkey_class.from_private_key_file(filename, password) + self._log(DEBUG, 'Trying discovered key %s in %s' % (hexlify(key.get_fingerprint()), filename)) + # for 2-factor auth a successfully auth'd key will result in ['password'] + allowed_types = self._transport.auth_publickey(username, key) + two_factor = (allowed_types == ['password']) + if not two_factor: + return + break + except SSHException, e: + saved_exception = e + except IOError, e: + saved_exception = e + + if password is not None: + try: + self._transport.auth_password(username, password) + return + except SSHException, e: + saved_exception = e + elif two_factor: + raise SSHException('Two-factor authentication requires a password') + + # if we got an auth-failed exception earlier, re-raise it + if saved_exception is not None: + raise saved_exception + raise SSHException('No authentication methods available') + + def _log(self, level, msg): + self._transport._log(level, msg) + diff --git a/contrib/site-packages/paramiko/common.py b/contrib/site-packages/paramiko/common.py new file mode 100644 index 00000000..3d7ca588 --- /dev/null +++ b/contrib/site-packages/paramiko/common.py @@ -0,0 +1,129 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +Common constants and global variables. +""" + +MSG_DISCONNECT, MSG_IGNORE, MSG_UNIMPLEMENTED, MSG_DEBUG, MSG_SERVICE_REQUEST, \ + MSG_SERVICE_ACCEPT = range(1, 7) +MSG_KEXINIT, MSG_NEWKEYS = range(20, 22) +MSG_USERAUTH_REQUEST, MSG_USERAUTH_FAILURE, MSG_USERAUTH_SUCCESS, \ + MSG_USERAUTH_BANNER = range(50, 54) +MSG_USERAUTH_PK_OK = 60 +MSG_USERAUTH_INFO_REQUEST, MSG_USERAUTH_INFO_RESPONSE = range(60, 62) +MSG_GLOBAL_REQUEST, MSG_REQUEST_SUCCESS, MSG_REQUEST_FAILURE = range(80, 83) +MSG_CHANNEL_OPEN, MSG_CHANNEL_OPEN_SUCCESS, MSG_CHANNEL_OPEN_FAILURE, \ + MSG_CHANNEL_WINDOW_ADJUST, MSG_CHANNEL_DATA, MSG_CHANNEL_EXTENDED_DATA, \ + MSG_CHANNEL_EOF, MSG_CHANNEL_CLOSE, MSG_CHANNEL_REQUEST, \ + MSG_CHANNEL_SUCCESS, MSG_CHANNEL_FAILURE = range(90, 101) + + +# for debugging: +MSG_NAMES = { + MSG_DISCONNECT: 'disconnect', + MSG_IGNORE: 'ignore', + MSG_UNIMPLEMENTED: 'unimplemented', + MSG_DEBUG: 'debug', + MSG_SERVICE_REQUEST: 'service-request', + MSG_SERVICE_ACCEPT: 'service-accept', + MSG_KEXINIT: 'kexinit', + MSG_NEWKEYS: 'newkeys', + 30: 'kex30', + 31: 'kex31', + 32: 'kex32', + 33: 'kex33', + 34: 'kex34', + MSG_USERAUTH_REQUEST: 'userauth-request', + MSG_USERAUTH_FAILURE: 'userauth-failure', + MSG_USERAUTH_SUCCESS: 'userauth-success', + MSG_USERAUTH_BANNER: 'userauth--banner', + MSG_USERAUTH_PK_OK: 'userauth-60(pk-ok/info-request)', + MSG_USERAUTH_INFO_RESPONSE: 'userauth-info-response', + MSG_GLOBAL_REQUEST: 'global-request', + MSG_REQUEST_SUCCESS: 'request-success', + MSG_REQUEST_FAILURE: 'request-failure', + MSG_CHANNEL_OPEN: 'channel-open', + MSG_CHANNEL_OPEN_SUCCESS: 'channel-open-success', + MSG_CHANNEL_OPEN_FAILURE: 'channel-open-failure', + MSG_CHANNEL_WINDOW_ADJUST: 'channel-window-adjust', + MSG_CHANNEL_DATA: 'channel-data', + MSG_CHANNEL_EXTENDED_DATA: 'channel-extended-data', + MSG_CHANNEL_EOF: 'channel-eof', + MSG_CHANNEL_CLOSE: 'channel-close', + MSG_CHANNEL_REQUEST: 'channel-request', + MSG_CHANNEL_SUCCESS: 'channel-success', + MSG_CHANNEL_FAILURE: 'channel-failure' + } + + +# authentication request return codes: +AUTH_SUCCESSFUL, AUTH_PARTIALLY_SUCCESSFUL, AUTH_FAILED = range(3) + + +# channel request failed reasons: +(OPEN_SUCCEEDED, + OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED, + OPEN_FAILED_CONNECT_FAILED, + OPEN_FAILED_UNKNOWN_CHANNEL_TYPE, + OPEN_FAILED_RESOURCE_SHORTAGE) = range(0, 5) + + +CONNECTION_FAILED_CODE = { + 1: 'Administratively prohibited', + 2: 'Connect failed', + 3: 'Unknown channel type', + 4: 'Resource shortage' +} + + +DISCONNECT_SERVICE_NOT_AVAILABLE, DISCONNECT_AUTH_CANCELLED_BY_USER, \ + DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE = 7, 13, 14 + +from Crypto import Random + +# keep a crypto-strong PRNG nearby +rng = Random.new() + +import sys +if sys.version_info < (2, 3): + try: + import logging + except: + import logging22 as logging + import select + PY22 = True + + import socket + if not hasattr(socket, 'timeout'): + class timeout(socket.error): pass + socket.timeout = timeout + del timeout +else: + import logging + PY22 = False + + +DEBUG = logging.DEBUG +INFO = logging.INFO +WARNING = logging.WARNING +ERROR = logging.ERROR +CRITICAL = logging.CRITICAL + +# Common IO/select/etc sleep period, in seconds +io_sleep = 0.01 diff --git a/contrib/site-packages/paramiko/compress.py b/contrib/site-packages/paramiko/compress.py new file mode 100644 index 00000000..b55f0b1d --- /dev/null +++ b/contrib/site-packages/paramiko/compress.py @@ -0,0 +1,39 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +Compression implementations for a Transport. +""" + +import zlib + + +class ZlibCompressor (object): + def __init__(self): + self.z = zlib.compressobj(9) + + def __call__(self, data): + return self.z.compress(data) + self.z.flush(zlib.Z_FULL_FLUSH) + + +class ZlibDecompressor (object): + def __init__(self): + self.z = zlib.decompressobj() + + def __call__(self, data): + return self.z.decompress(data) diff --git a/contrib/site-packages/paramiko/config.py b/contrib/site-packages/paramiko/config.py new file mode 100644 index 00000000..1705de76 --- /dev/null +++ b/contrib/site-packages/paramiko/config.py @@ -0,0 +1,266 @@ +# Copyright (C) 2006-2007 Robey Pointer +# Copyright (C) 2012 Olle Lundberg +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +L{SSHConfig}. +""" + +import fnmatch +import os +import re +import socket + +SSH_PORT = 22 +proxy_re = re.compile(r"^(proxycommand)\s*=*\s*(.*)", re.I) + + +class LazyFqdn(object): + """ + Returns the host's fqdn on request as string. + """ + + def __init__(self, config, host=None): + self.fqdn = None + self.config = config + self.host = host + + def __str__(self): + if self.fqdn is None: + # + # If the SSH config contains AddressFamily, use that when + # determining the local host's FQDN. Using socket.getfqdn() from + # the standard library is the most general solution, but can + # result in noticeable delays on some platforms when IPv6 is + # misconfigured or not available, as it calls getaddrinfo with no + # address family specified, so both IPv4 and IPv6 are checked. + # + + # Handle specific option + fqdn = None + address_family = self.config.get('addressfamily', 'any').lower() + if address_family != 'any': + try: + family = socket.AF_INET if address_family == 'inet' \ + else socket.AF_INET6 + results = socket.getaddrinfo( + self.host, + None, + family, + socket.SOCK_DGRAM, + socket.IPPROTO_IP, + socket.AI_CANONNAME + ) + for res in results: + af, socktype, proto, canonname, sa = res + if canonname and '.' in canonname: + fqdn = canonname + break + # giaerror -> socket.getaddrinfo() can't resolve self.host + # (which is from socket.gethostname()). Fall back to the + # getfqdn() call below. + except socket.gaierror: + pass + # Handle 'any' / unspecified + if fqdn is None: + fqdn = socket.getfqdn() + # Cache + self.fqdn = fqdn + return self.fqdn + + +class SSHConfig (object): + """ + Representation of config information as stored in the format used by + OpenSSH. Queries can be made via L{lookup}. The format is described in + OpenSSH's C{ssh_config} man page. This class is provided primarily as a + convenience to posix users (since the OpenSSH format is a de-facto + standard on posix) but should work fine on Windows too. + + @since: 1.6 + """ + + def __init__(self): + """ + Create a new OpenSSH config object. + """ + self._config = [] + + def parse(self, file_obj): + """ + Read an OpenSSH config from the given file object. + + @param file_obj: a file-like object to read the config file from + @type file_obj: file + """ + host = {"host": ['*'], "config": {}} + for line in file_obj: + line = line.rstrip('\n').lstrip() + if (line == '') or (line[0] == '#'): + continue + if '=' in line: + # Ensure ProxyCommand gets properly split + if line.lower().strip().startswith('proxycommand'): + match = proxy_re.match(line) + key, value = match.group(1).lower(), match.group(2) + else: + key, value = line.split('=', 1) + key = key.strip().lower() + else: + # find first whitespace, and split there + i = 0 + while (i < len(line)) and not line[i].isspace(): + i += 1 + if i == len(line): + raise Exception('Unparsable line: %r' % line) + key = line[:i].lower() + value = line[i:].lstrip() + + if key == 'host': + self._config.append(host) + value = value.split() + host = {key: value, 'config': {}} + #identityfile, localforward, remoteforward keys are special cases, since they are allowed to be + # specified multiple times and they should be tried in order + # of specification. + + elif key in ['identityfile', 'localforward', 'remoteforward']: + if key in host['config']: + host['config'][key].append(value) + else: + host['config'][key] = [value] + elif key not in host['config']: + host['config'].update({key: value}) + self._config.append(host) + + def lookup(self, hostname): + """ + Return a dict of config options for a given hostname. + + The host-matching rules of OpenSSH's C{ssh_config} man page are used, + which means that all configuration options from matching host + specifications are merged, with more specific hostmasks taking + precedence. In other words, if C{"Port"} is set under C{"Host *"} + and also C{"Host *.example.com"}, and the lookup is for + C{"ssh.example.com"}, then the port entry for C{"Host *.example.com"} + will win out. + + The keys in the returned dict are all normalized to lowercase (look for + C{"port"}, not C{"Port"}. The values are processed according to the + rules for substitution variable expansion in C{ssh_config}. + + @param hostname: the hostname to lookup + @type hostname: str + """ + + matches = [config for config in self._config if + self._allowed(hostname, config['host'])] + + ret = {} + for match in matches: + for key, value in match['config'].iteritems(): + if key not in ret: + # Create a copy of the original value, + # else it will reference the original list + # in self._config and update that value too + # when the extend() is being called. + ret[key] = value[:] + elif key == 'identityfile': + ret[key].extend(value) + ret = self._expand_variables(ret, hostname) + return ret + + def _allowed(self, hostname, hosts): + match = False + for host in hosts: + if host.startswith('!') and fnmatch.fnmatch(hostname, host[1:]): + return False + elif fnmatch.fnmatch(hostname, host): + match = True + return match + + def _expand_variables(self, config, hostname): + """ + Return a dict of config options with expanded substitutions + for a given hostname. + + Please refer to man C{ssh_config} for the parameters that + are replaced. + + @param config: the config for the hostname + @type hostname: dict + @param hostname: the hostname that the config belongs to + @type hostname: str + """ + + if 'hostname' in config: + config['hostname'] = config['hostname'].replace('%h', hostname) + else: + config['hostname'] = hostname + + if 'port' in config: + port = config['port'] + else: + port = SSH_PORT + + user = os.getenv('USER') + if 'user' in config: + remoteuser = config['user'] + else: + remoteuser = user + + host = socket.gethostname().split('.')[0] + fqdn = LazyFqdn(config, host) + homedir = os.path.expanduser('~') + replacements = {'controlpath': + [ + ('%h', config['hostname']), + ('%l', fqdn), + ('%L', host), + ('%n', hostname), + ('%p', port), + ('%r', remoteuser), + ('%u', user) + ], + 'identityfile': + [ + ('~', homedir), + ('%d', homedir), + ('%h', config['hostname']), + ('%l', fqdn), + ('%u', user), + ('%r', remoteuser) + ], + 'proxycommand': + [ + ('%h', config['hostname']), + ('%p', port), + ('%r', remoteuser) + ] + } + + for k in config: + if k in replacements: + for find, replace in replacements[k]: + if isinstance(config[k], list): + for item in range(len(config[k])): + config[k][item] = config[k][item].\ + replace(find, str(replace)) + else: + config[k] = config[k].replace(find, str(replace)) + return config diff --git a/contrib/site-packages/paramiko/dsskey.py b/contrib/site-packages/paramiko/dsskey.py new file mode 100644 index 00000000..f6ecb2a7 --- /dev/null +++ b/contrib/site-packages/paramiko/dsskey.py @@ -0,0 +1,196 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +L{DSSKey} +""" + +from Crypto.PublicKey import DSA +from Crypto.Hash import SHA + +from paramiko.common import * +from paramiko import util +from paramiko.ssh_exception import SSHException +from paramiko.message import Message +from paramiko.ber import BER, BERException +from paramiko.pkey import PKey + + +class DSSKey (PKey): + """ + Representation of a DSS key which can be used to sign an verify SSH2 + data. + """ + + def __init__(self, msg=None, data=None, filename=None, password=None, vals=None, file_obj=None): + self.p = None + self.q = None + self.g = None + self.y = None + self.x = None + if file_obj is not None: + self._from_private_key(file_obj, password) + return + if filename is not None: + self._from_private_key_file(filename, password) + return + if (msg is None) and (data is not None): + msg = Message(data) + if vals is not None: + self.p, self.q, self.g, self.y = vals + else: + if msg is None: + raise SSHException('Key object may not be empty') + if msg.get_string() != 'ssh-dss': + raise SSHException('Invalid key') + self.p = msg.get_mpint() + self.q = msg.get_mpint() + self.g = msg.get_mpint() + self.y = msg.get_mpint() + self.size = util.bit_length(self.p) + + def __str__(self): + m = Message() + m.add_string('ssh-dss') + m.add_mpint(self.p) + m.add_mpint(self.q) + m.add_mpint(self.g) + m.add_mpint(self.y) + return str(m) + + def __hash__(self): + h = hash(self.get_name()) + h = h * 37 + hash(self.p) + h = h * 37 + hash(self.q) + h = h * 37 + hash(self.g) + h = h * 37 + hash(self.y) + # h might be a long by now... + return hash(h) + + def get_name(self): + return 'ssh-dss' + + def get_bits(self): + return self.size + + def can_sign(self): + return self.x is not None + + def sign_ssh_data(self, rng, data): + digest = SHA.new(data).digest() + dss = DSA.construct((long(self.y), long(self.g), long(self.p), long(self.q), long(self.x))) + # generate a suitable k + qsize = len(util.deflate_long(self.q, 0)) + while True: + k = util.inflate_long(rng.read(qsize), 1) + if (k > 2) and (k < self.q): + break + r, s = dss.sign(util.inflate_long(digest, 1), k) + m = Message() + m.add_string('ssh-dss') + # apparently, in rare cases, r or s may be shorter than 20 bytes! + rstr = util.deflate_long(r, 0) + sstr = util.deflate_long(s, 0) + if len(rstr) < 20: + rstr = '\x00' * (20 - len(rstr)) + rstr + if len(sstr) < 20: + sstr = '\x00' * (20 - len(sstr)) + sstr + m.add_string(rstr + sstr) + return m + + def verify_ssh_sig(self, data, msg): + if len(str(msg)) == 40: + # spies.com bug: signature has no header + sig = str(msg) + else: + kind = msg.get_string() + if kind != 'ssh-dss': + return 0 + sig = msg.get_string() + + # pull out (r, s) which are NOT encoded as mpints + sigR = util.inflate_long(sig[:20], 1) + sigS = util.inflate_long(sig[20:], 1) + sigM = util.inflate_long(SHA.new(data).digest(), 1) + + dss = DSA.construct((long(self.y), long(self.g), long(self.p), long(self.q))) + return dss.verify(sigM, (sigR, sigS)) + + def _encode_key(self): + if self.x is None: + raise SSHException('Not enough key information') + keylist = [ 0, self.p, self.q, self.g, self.y, self.x ] + try: + b = BER() + b.encode(keylist) + except BERException: + raise SSHException('Unable to create ber encoding of key') + return str(b) + + def write_private_key_file(self, filename, password=None): + self._write_private_key_file('DSA', filename, self._encode_key(), password) + + def write_private_key(self, file_obj, password=None): + self._write_private_key('DSA', file_obj, self._encode_key(), password) + + def generate(bits=1024, progress_func=None): + """ + Generate a new private DSS key. This factory function can be used to + generate a new host key or authentication key. + + @param bits: number of bits the generated key should be. + @type bits: int + @param progress_func: an optional function to call at key points in + key generation (used by C{pyCrypto.PublicKey}). + @type progress_func: function + @return: new private key + @rtype: L{DSSKey} + """ + dsa = DSA.generate(bits, rng.read, progress_func) + key = DSSKey(vals=(dsa.p, dsa.q, dsa.g, dsa.y)) + key.x = dsa.x + return key + generate = staticmethod(generate) + + + ### internals... + + + def _from_private_key_file(self, filename, password): + data = self._read_private_key_file('DSA', filename, password) + self._decode_key(data) + + def _from_private_key(self, file_obj, password): + data = self._read_private_key('DSA', file_obj, password) + self._decode_key(data) + + def _decode_key(self, data): + # private key file contains: + # DSAPrivateKey = { version = 0, p, q, g, y, x } + try: + keylist = BER(data).decode() + except BERException, x: + raise SSHException('Unable to parse key file: ' + str(x)) + if (type(keylist) is not list) or (len(keylist) < 6) or (keylist[0] != 0): + raise SSHException('not a valid DSA private key file (bad ber encoding)') + self.p = keylist[1] + self.q = keylist[2] + self.g = keylist[3] + self.y = keylist[4] + self.x = keylist[5] + self.size = util.bit_length(self.p) diff --git a/contrib/site-packages/paramiko/ecdsakey.py b/contrib/site-packages/paramiko/ecdsakey.py new file mode 100644 index 00000000..ac840ab7 --- /dev/null +++ b/contrib/site-packages/paramiko/ecdsakey.py @@ -0,0 +1,181 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distrubuted in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +L{ECDSAKey} +""" + +import binascii +from ecdsa import SigningKey, VerifyingKey, der, curves +from ecdsa.util import number_to_string, sigencode_string, sigencode_strings, sigdecode_strings +from Crypto.Hash import SHA256, MD5 +from Crypto.Cipher import DES3 + +from paramiko.common import * +from paramiko import util +from paramiko.message import Message +from paramiko.ber import BER, BERException +from paramiko.pkey import PKey +from paramiko.ssh_exception import SSHException + + +class ECDSAKey (PKey): + """ + Representation of an ECDSA key which can be used to sign and verify SSH2 + data. + """ + + def __init__(self, msg=None, data=None, filename=None, password=None, vals=None, file_obj=None): + self.verifying_key = None + self.signing_key = None + if file_obj is not None: + self._from_private_key(file_obj, password) + return + if filename is not None: + self._from_private_key_file(filename, password) + return + if (msg is None) and (data is not None): + msg = Message(data) + if vals is not None: + self.verifying_key, self.signing_key = vals + else: + if msg is None: + raise SSHException('Key object may not be empty') + if msg.get_string() != 'ecdsa-sha2-nistp256': + raise SSHException('Invalid key') + curvename = msg.get_string() + if curvename != 'nistp256': + raise SSHException("Can't handle curve of type %s" % curvename) + + pointinfo = msg.get_string() + if pointinfo[0] != "\x04": + raise SSHException('Point compression is being used: %s'% + binascii.hexlify(pointinfo)) + self.verifying_key = VerifyingKey.from_string(pointinfo[1:], + curve=curves.NIST256p) + self.size = 256 + + def __str__(self): + key = self.verifying_key + m = Message() + m.add_string('ecdsa-sha2-nistp256') + m.add_string('nistp256') + + point_str = "\x04" + key.to_string() + + m.add_string(point_str) + return str(m) + + def __hash__(self): + h = hash(self.get_name()) + h = h * 37 + hash(self.verifying_key.pubkey.point.x()) + h = h * 37 + hash(self.verifying_key.pubkey.point.y()) + return hash(h) + + def get_name(self): + return 'ecdsa-sha2-nistp256' + + def get_bits(self): + return self.size + + def can_sign(self): + return self.signing_key is not None + + def sign_ssh_data(self, rpool, data): + digest = SHA256.new(data).digest() + sig = self.signing_key.sign_digest(digest, entropy=rpool.read, + sigencode=self._sigencode) + m = Message() + m.add_string('ecdsa-sha2-nistp256') + m.add_string(sig) + return m + + def verify_ssh_sig(self, data, msg): + if msg.get_string() != 'ecdsa-sha2-nistp256': + return False + sig = msg.get_string() + + # verify the signature by SHA'ing the data and encrypting it + # using the public key. + hash_obj = SHA256.new(data).digest() + return self.verifying_key.verify_digest(sig, hash_obj, + sigdecode=self._sigdecode) + + def write_private_key_file(self, filename, password=None): + key = self.signing_key or self.verifying_key + self._write_private_key_file('EC', filename, key.to_der(), password) + + def write_private_key(self, file_obj, password=None): + key = self.signing_key or self.verifying_key + self._write_private_key('EC', file_obj, key.to_der(), password) + + def generate(bits, progress_func=None): + """ + Generate a new private RSA key. This factory function can be used to + generate a new host key or authentication key. + + @param bits: number of bits the generated key should be. + @type bits: int + @param progress_func: an optional function to call at key points in + key generation (used by C{pyCrypto.PublicKey}). + @type progress_func: function + @return: new private key + @rtype: L{RSAKey} + """ + signing_key = ECDSA.generate() + key = ECDSAKey(vals=(signing_key, signing_key.get_verifying_key())) + return key + generate = staticmethod(generate) + + + ### internals... + + + def _from_private_key_file(self, filename, password): + data = self._read_private_key_file('EC', filename, password) + self._decode_key(data) + + def _from_private_key(self, file_obj, password): + data = self._read_private_key('EC', file_obj, password) + self._decode_key(data) + + ALLOWED_PADDINGS = ['\x01', '\x02\x02', '\x03\x03\x03', '\x04\x04\x04\x04', + '\x05\x05\x05\x05\x05', '\x06\x06\x06\x06\x06\x06', + '\x07\x07\x07\x07\x07\x07\x07'] + def _decode_key(self, data): + s, padding = der.remove_sequence(data) + if padding: + if padding not in self.ALLOWED_PADDINGS: + raise ValueError, "weird padding: %s" % (binascii.hexlify(empty)) + data = data[:-len(padding)] + key = SigningKey.from_der(data) + self.signing_key = key + self.verifying_key = key.get_verifying_key() + self.size = 256 + + def _sigencode(self, r, s, order): + msg = Message() + msg.add_mpint(r) + msg.add_mpint(s) + return str(msg) + + def _sigdecode(self, sig, order): + msg = Message(sig) + r = msg.get_mpint() + s = msg.get_mpint() + return (r, s) diff --git a/contrib/site-packages/paramiko/file.py b/contrib/site-packages/paramiko/file.py new file mode 100644 index 00000000..5fd81cfe --- /dev/null +++ b/contrib/site-packages/paramiko/file.py @@ -0,0 +1,460 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +BufferedFile. +""" + +from cStringIO import StringIO + + +class BufferedFile (object): + """ + Reusable base class to implement python-style file buffering around a + simpler stream. + """ + + _DEFAULT_BUFSIZE = 8192 + + SEEK_SET = 0 + SEEK_CUR = 1 + SEEK_END = 2 + + FLAG_READ = 0x1 + FLAG_WRITE = 0x2 + FLAG_APPEND = 0x4 + FLAG_BINARY = 0x10 + FLAG_BUFFERED = 0x20 + FLAG_LINE_BUFFERED = 0x40 + FLAG_UNIVERSAL_NEWLINE = 0x80 + + def __init__(self): + self.newlines = None + self._flags = 0 + self._bufsize = self._DEFAULT_BUFSIZE + self._wbuffer = StringIO() + self._rbuffer = '' + self._at_trailing_cr = False + self._closed = False + # pos - position within the file, according to the user + # realpos - position according the OS + # (these may be different because we buffer for line reading) + self._pos = self._realpos = 0 + # size only matters for seekable files + self._size = 0 + + def __del__(self): + self.close() + + def __iter__(self): + """ + Returns an iterator that can be used to iterate over the lines in this + file. This iterator happens to return the file itself, since a file is + its own iterator. + + @raise ValueError: if the file is closed. + + @return: an interator. + @rtype: iterator + """ + if self._closed: + raise ValueError('I/O operation on closed file') + return self + + def close(self): + """ + Close the file. Future read and write operations will fail. + """ + self.flush() + self._closed = True + + def flush(self): + """ + Write out any data in the write buffer. This may do nothing if write + buffering is not turned on. + """ + self._write_all(self._wbuffer.getvalue()) + self._wbuffer = StringIO() + return + + def next(self): + """ + Returns the next line from the input, or raises L{StopIteration} when + EOF is hit. Unlike python file objects, it's okay to mix calls to + C{next} and L{readline}. + + @raise StopIteration: when the end of the file is reached. + + @return: a line read from the file. + @rtype: str + """ + line = self.readline() + if not line: + raise StopIteration + return line + + def read(self, size=None): + """ + Read at most C{size} bytes from the file (less if we hit the end of the + file first). If the C{size} argument is negative or omitted, read all + the remaining data in the file. + + @param size: maximum number of bytes to read + @type size: int + @return: data read from the file, or an empty string if EOF was + encountered immediately + @rtype: str + """ + if self._closed: + raise IOError('File is closed') + if not (self._flags & self.FLAG_READ): + raise IOError('File is not open for reading') + if (size is None) or (size < 0): + # go for broke + result = self._rbuffer + self._rbuffer = '' + self._pos += len(result) + while True: + try: + new_data = self._read(self._DEFAULT_BUFSIZE) + except EOFError: + new_data = None + if (new_data is None) or (len(new_data) == 0): + break + result += new_data + self._realpos += len(new_data) + self._pos += len(new_data) + return result + if size <= len(self._rbuffer): + result = self._rbuffer[:size] + self._rbuffer = self._rbuffer[size:] + self._pos += len(result) + return result + while len(self._rbuffer) < size: + read_size = size - len(self._rbuffer) + if self._flags & self.FLAG_BUFFERED: + read_size = max(self._bufsize, read_size) + try: + new_data = self._read(read_size) + except EOFError: + new_data = None + if (new_data is None) or (len(new_data) == 0): + break + self._rbuffer += new_data + self._realpos += len(new_data) + result = self._rbuffer[:size] + self._rbuffer = self._rbuffer[size:] + self._pos += len(result) + return result + + def readline(self, size=None): + """ + Read one entire line from the file. A trailing newline character is + kept in the string (but may be absent when a file ends with an + incomplete line). If the size argument is present and non-negative, it + is a maximum byte count (including the trailing newline) and an + incomplete line may be returned. An empty string is returned only when + EOF is encountered immediately. + + @note: Unlike stdio's C{fgets()}, the returned string contains null + characters (C{'\\0'}) if they occurred in the input. + + @param size: maximum length of returned string. + @type size: int + @return: next line of the file, or an empty string if the end of the + file has been reached. + @rtype: str + """ + # it's almost silly how complex this function is. + if self._closed: + raise IOError('File is closed') + if not (self._flags & self.FLAG_READ): + raise IOError('File not open for reading') + line = self._rbuffer + while True: + if self._at_trailing_cr and (self._flags & self.FLAG_UNIVERSAL_NEWLINE) and (len(line) > 0): + # edge case: the newline may be '\r\n' and we may have read + # only the first '\r' last time. + if line[0] == '\n': + line = line[1:] + self._record_newline('\r\n') + else: + self._record_newline('\r') + self._at_trailing_cr = False + # check size before looking for a linefeed, in case we already have + # enough. + if (size is not None) and (size >= 0): + if len(line) >= size: + # truncate line and return + self._rbuffer = line[size:] + line = line[:size] + self._pos += len(line) + return line + n = size - len(line) + else: + n = self._bufsize + if ('\n' in line) or ((self._flags & self.FLAG_UNIVERSAL_NEWLINE) and ('\r' in line)): + break + try: + new_data = self._read(n) + except EOFError: + new_data = None + if (new_data is None) or (len(new_data) == 0): + self._rbuffer = '' + self._pos += len(line) + return line + line += new_data + self._realpos += len(new_data) + # find the newline + pos = line.find('\n') + if self._flags & self.FLAG_UNIVERSAL_NEWLINE: + rpos = line.find('\r') + if (rpos >= 0) and ((rpos < pos) or (pos < 0)): + pos = rpos + xpos = pos + 1 + if (line[pos] == '\r') and (xpos < len(line)) and (line[xpos] == '\n'): + xpos += 1 + self._rbuffer = line[xpos:] + lf = line[pos:xpos] + line = line[:pos] + '\n' + if (len(self._rbuffer) == 0) and (lf == '\r'): + # we could read the line up to a '\r' and there could still be a + # '\n' following that we read next time. note that and eat it. + self._at_trailing_cr = True + else: + self._record_newline(lf) + self._pos += len(line) + return line + + def readlines(self, sizehint=None): + """ + Read all remaining lines using L{readline} and return them as a list. + If the optional C{sizehint} argument is present, instead of reading up + to EOF, whole lines totalling approximately sizehint bytes (possibly + after rounding up to an internal buffer size) are read. + + @param sizehint: desired maximum number of bytes to read. + @type sizehint: int + @return: list of lines read from the file. + @rtype: list + """ + lines = [] + bytes = 0 + while True: + line = self.readline() + if len(line) == 0: + break + lines.append(line) + bytes += len(line) + if (sizehint is not None) and (bytes >= sizehint): + break + return lines + + def seek(self, offset, whence=0): + """ + Set the file's current position, like stdio's C{fseek}. Not all file + objects support seeking. + + @note: If a file is opened in append mode (C{'a'} or C{'a+'}), any seek + operations will be undone at the next write (as the file position + will move back to the end of the file). + + @param offset: position to move to within the file, relative to + C{whence}. + @type offset: int + @param whence: type of movement: 0 = absolute; 1 = relative to the + current position; 2 = relative to the end of the file. + @type whence: int + + @raise IOError: if the file doesn't support random access. + """ + raise IOError('File does not support seeking.') + + def tell(self): + """ + Return the file's current position. This may not be accurate or + useful if the underlying file doesn't support random access, or was + opened in append mode. + + @return: file position (in bytes). + @rtype: int + """ + return self._pos + + def write(self, data): + """ + Write data to the file. If write buffering is on (C{bufsize} was + specified and non-zero), some or all of the data may not actually be + written yet. (Use L{flush} or L{close} to force buffered data to be + written out.) + + @param data: data to write. + @type data: str + """ + if self._closed: + raise IOError('File is closed') + if not (self._flags & self.FLAG_WRITE): + raise IOError('File not open for writing') + if not (self._flags & self.FLAG_BUFFERED): + self._write_all(data) + return + self._wbuffer.write(data) + if self._flags & self.FLAG_LINE_BUFFERED: + # only scan the new data for linefeed, to avoid wasting time. + last_newline_pos = data.rfind('\n') + if last_newline_pos >= 0: + wbuf = self._wbuffer.getvalue() + last_newline_pos += len(wbuf) - len(data) + self._write_all(wbuf[:last_newline_pos + 1]) + self._wbuffer = StringIO() + self._wbuffer.write(wbuf[last_newline_pos + 1:]) + return + # even if we're line buffering, if the buffer has grown past the + # buffer size, force a flush. + if self._wbuffer.tell() >= self._bufsize: + self.flush() + return + + def writelines(self, sequence): + """ + Write a sequence of strings to the file. The sequence can be any + iterable object producing strings, typically a list of strings. (The + name is intended to match L{readlines}; C{writelines} does not add line + separators.) + + @param sequence: an iterable sequence of strings. + @type sequence: sequence + """ + for line in sequence: + self.write(line) + return + + def xreadlines(self): + """ + Identical to C{iter(f)}. This is a deprecated file interface that + predates python iterator support. + + @return: an iterator. + @rtype: iterator + """ + return self + + @property + def closed(self): + return self._closed + + + ### overrides... + + + def _read(self, size): + """ + I{(subclass override)} + Read data from the stream. Return C{None} or raise C{EOFError} to + indicate EOF. + """ + raise EOFError() + + def _write(self, data): + """ + I{(subclass override)} + Write data into the stream. + """ + raise IOError('write not implemented') + + def _get_size(self): + """ + I{(subclass override)} + Return the size of the file. This is called from within L{_set_mode} + if the file is opened in append mode, so the file position can be + tracked and L{seek} and L{tell} will work correctly. If the file is + a stream that can't be randomly accessed, you don't need to override + this method, + """ + return 0 + + + ### internals... + + + def _set_mode(self, mode='r', bufsize=-1): + """ + Subclasses call this method to initialize the BufferedFile. + """ + # set bufsize in any event, because it's used for readline(). + self._bufsize = self._DEFAULT_BUFSIZE + if bufsize < 0: + # do no buffering by default, because otherwise writes will get + # buffered in a way that will probably confuse people. + bufsize = 0 + if bufsize == 1: + # apparently, line buffering only affects writes. reads are only + # buffered if you call readline (directly or indirectly: iterating + # over a file will indirectly call readline). + self._flags |= self.FLAG_BUFFERED | self.FLAG_LINE_BUFFERED + elif bufsize > 1: + self._bufsize = bufsize + self._flags |= self.FLAG_BUFFERED + self._flags &= ~self.FLAG_LINE_BUFFERED + elif bufsize == 0: + # unbuffered + self._flags &= ~(self.FLAG_BUFFERED | self.FLAG_LINE_BUFFERED) + + if ('r' in mode) or ('+' in mode): + self._flags |= self.FLAG_READ + if ('w' in mode) or ('+' in mode): + self._flags |= self.FLAG_WRITE + if ('a' in mode): + self._flags |= self.FLAG_WRITE | self.FLAG_APPEND + self._size = self._get_size() + self._pos = self._realpos = self._size + if ('b' in mode): + self._flags |= self.FLAG_BINARY + if ('U' in mode): + self._flags |= self.FLAG_UNIVERSAL_NEWLINE + # built-in file objects have this attribute to store which kinds of + # line terminations they've seen: + # + self.newlines = None + + def _write_all(self, data): + # the underlying stream may be something that does partial writes (like + # a socket). + while len(data) > 0: + count = self._write(data) + data = data[count:] + if self._flags & self.FLAG_APPEND: + self._size += count + self._pos = self._realpos = self._size + else: + self._pos += count + self._realpos += count + return None + + def _record_newline(self, newline): + # silliness about tracking what kinds of newlines we've seen. + # i don't understand why it can be None, a string, or a tuple, instead + # of just always being a tuple, but we'll emulate that behavior anyway. + if not (self._flags & self.FLAG_UNIVERSAL_NEWLINE): + return + if self.newlines is None: + self.newlines = newline + elif (type(self.newlines) is str) and (self.newlines != newline): + self.newlines = (self.newlines, newline) + elif newline not in self.newlines: + self.newlines += (newline,) diff --git a/contrib/site-packages/paramiko/hostkeys.py b/contrib/site-packages/paramiko/hostkeys.py new file mode 100644 index 00000000..9bcf0d55 --- /dev/null +++ b/contrib/site-packages/paramiko/hostkeys.py @@ -0,0 +1,342 @@ +# Copyright (C) 2006-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +L{HostKeys} +""" + +import base64 +import binascii +from Crypto.Hash import SHA, HMAC +import UserDict + +from paramiko.common import * +from paramiko.dsskey import DSSKey +from paramiko.rsakey import RSAKey +from paramiko.util import get_logger +from paramiko.ecdsakey import ECDSAKey + + +class InvalidHostKey(Exception): + + def __init__(self, line, exc): + self.line = line + self.exc = exc + self.args = (line, exc) + + +class HostKeyEntry: + """ + Representation of a line in an OpenSSH-style "known hosts" file. + """ + + def __init__(self, hostnames=None, key=None): + self.valid = (hostnames is not None) and (key is not None) + self.hostnames = hostnames + self.key = key + + def from_line(cls, line, lineno=None): + """ + Parses the given line of text to find the names for the host, + the type of key, and the key data. The line is expected to be in the + format used by the openssh known_hosts file. + + Lines are expected to not have leading or trailing whitespace. + We don't bother to check for comments or empty lines. All of + that should be taken care of before sending the line to us. + + @param line: a line from an OpenSSH known_hosts file + @type line: str + """ + log = get_logger('paramiko.hostkeys') + fields = line.split(' ') + if len(fields) < 3: + # Bad number of fields + log.info("Not enough fields found in known_hosts in line %s (%r)" % + (lineno, line)) + return None + fields = fields[:3] + + names, keytype, key = fields + names = names.split(',') + + # Decide what kind of key we're looking at and create an object + # to hold it accordingly. + try: + if keytype == 'ssh-rsa': + key = RSAKey(data=base64.decodestring(key)) + elif keytype == 'ssh-dss': + key = DSSKey(data=base64.decodestring(key)) + elif keytype == 'ecdsa-sha2-nistp256': + key = ECDSAKey(data=base64.decodestring(key)) + else: + log.info("Unable to handle key of type %s" % (keytype,)) + return None + + except binascii.Error, e: + raise InvalidHostKey(line, e) + + return cls(names, key) + from_line = classmethod(from_line) + + def to_line(self): + """ + Returns a string in OpenSSH known_hosts file format, or None if + the object is not in a valid state. A trailing newline is + included. + """ + if self.valid: + return '%s %s %s\n' % (','.join(self.hostnames), self.key.get_name(), + self.key.get_base64()) + return None + + def __repr__(self): + return '' % (self.hostnames, self.key) + + +class HostKeys (UserDict.DictMixin): + """ + Representation of an openssh-style "known hosts" file. Host keys can be + read from one or more files, and then individual hosts can be looked up to + verify server keys during SSH negotiation. + + A HostKeys object can be treated like a dict; any dict lookup is equivalent + to calling L{lookup}. + + @since: 1.5.3 + """ + + def __init__(self, filename=None): + """ + Create a new HostKeys object, optionally loading keys from an openssh + style host-key file. + + @param filename: filename to load host keys from, or C{None} + @type filename: str + """ + # emulate a dict of { hostname: { keytype: PKey } } + self._entries = [] + if filename is not None: + self.load(filename) + + def add(self, hostname, keytype, key): + """ + Add a host key entry to the table. Any existing entry for a + C{(hostname, keytype)} pair will be replaced. + + @param hostname: the hostname (or IP) to add + @type hostname: str + @param keytype: key type (C{"ssh-rsa"} or C{"ssh-dss"}) + @type keytype: str + @param key: the key to add + @type key: L{PKey} + """ + for e in self._entries: + if (hostname in e.hostnames) and (e.key.get_name() == keytype): + e.key = key + return + self._entries.append(HostKeyEntry([hostname], key)) + + def load(self, filename): + """ + Read a file of known SSH host keys, in the format used by openssh. + This type of file unfortunately doesn't exist on Windows, but on + posix, it will usually be stored in + C{os.path.expanduser("~/.ssh/known_hosts")}. + + If this method is called multiple times, the host keys are merged, + not cleared. So multiple calls to C{load} will just call L{add}, + replacing any existing entries and adding new ones. + + @param filename: name of the file to read host keys from + @type filename: str + + @raise IOError: if there was an error reading the file + """ + f = open(filename, 'r') + for lineno, line in enumerate(f): + line = line.strip() + if (len(line) == 0) or (line[0] == '#'): + continue + e = HostKeyEntry.from_line(line, lineno) + if e is not None: + _hostnames = e.hostnames + for h in _hostnames: + if self.check(h, e.key): + e.hostnames.remove(h) + if len(e.hostnames): + self._entries.append(e) + f.close() + + def save(self, filename): + """ + Save host keys into a file, in the format used by openssh. The order of + keys in the file will be preserved when possible (if these keys were + loaded from a file originally). The single exception is that combined + lines will be split into individual key lines, which is arguably a bug. + + @param filename: name of the file to write + @type filename: str + + @raise IOError: if there was an error writing the file + + @since: 1.6.1 + """ + f = open(filename, 'w') + for e in self._entries: + line = e.to_line() + if line: + f.write(line) + f.close() + + def lookup(self, hostname): + """ + Find a hostkey entry for a given hostname or IP. If no entry is found, + C{None} is returned. Otherwise a dictionary of keytype to key is + returned. The keytype will be either C{"ssh-rsa"} or C{"ssh-dss"}. + + @param hostname: the hostname (or IP) to lookup + @type hostname: str + @return: keys associated with this host (or C{None}) + @rtype: dict(str, L{PKey}) + """ + class SubDict (UserDict.DictMixin): + def __init__(self, hostname, entries, hostkeys): + self._hostname = hostname + self._entries = entries + self._hostkeys = hostkeys + + def __getitem__(self, key): + for e in self._entries: + if e.key.get_name() == key: + return e.key + raise KeyError(key) + + def __setitem__(self, key, val): + for e in self._entries: + if e.key is None: + continue + if e.key.get_name() == key: + # replace + e.key = val + break + else: + # add a new one + e = HostKeyEntry([hostname], val) + self._entries.append(e) + self._hostkeys._entries.append(e) + + def keys(self): + return [e.key.get_name() for e in self._entries if e.key is not None] + + entries = [] + for e in self._entries: + for h in e.hostnames: + if (h.startswith('|1|') and (self.hash_host(hostname, h) == h)) or (h == hostname): + entries.append(e) + if len(entries) == 0: + return None + return SubDict(hostname, entries, self) + + def check(self, hostname, key): + """ + Return True if the given key is associated with the given hostname + in this dictionary. + + @param hostname: hostname (or IP) of the SSH server + @type hostname: str + @param key: the key to check + @type key: L{PKey} + @return: C{True} if the key is associated with the hostname; C{False} + if not + @rtype: bool + """ + k = self.lookup(hostname) + if k is None: + return False + host_key = k.get(key.get_name(), None) + if host_key is None: + return False + return str(host_key) == str(key) + + def clear(self): + """ + Remove all host keys from the dictionary. + """ + self._entries = [] + + def __getitem__(self, key): + ret = self.lookup(key) + if ret is None: + raise KeyError(key) + return ret + + def __setitem__(self, hostname, entry): + # don't use this please. + if len(entry) == 0: + self._entries.append(HostKeyEntry([hostname], None)) + return + for key_type in entry.keys(): + found = False + for e in self._entries: + if (hostname in e.hostnames) and (e.key.get_name() == key_type): + # replace + e.key = entry[key_type] + found = True + if not found: + self._entries.append(HostKeyEntry([hostname], entry[key_type])) + + def keys(self): + # python 2.4 sets would be nice here. + ret = [] + for e in self._entries: + for h in e.hostnames: + if h not in ret: + ret.append(h) + return ret + + def values(self): + ret = [] + for k in self.keys(): + ret.append(self.lookup(k)) + return ret + + def hash_host(hostname, salt=None): + """ + Return a "hashed" form of the hostname, as used by openssh when storing + hashed hostnames in the known_hosts file. + + @param hostname: the hostname to hash + @type hostname: str + @param salt: optional salt to use when hashing (must be 20 bytes long) + @type salt: str + @return: the hashed hostname + @rtype: str + """ + if salt is None: + salt = rng.read(SHA.digest_size) + else: + if salt.startswith('|1|'): + salt = salt.split('|')[2] + salt = base64.decodestring(salt) + assert len(salt) == SHA.digest_size + hmac = HMAC.HMAC(salt, hostname, SHA).digest() + hostkey = '|1|%s|%s' % (base64.encodestring(salt), base64.encodestring(hmac)) + return hostkey.replace('\n', '') + hash_host = staticmethod(hash_host) + diff --git a/contrib/site-packages/paramiko/kex_gex.py b/contrib/site-packages/paramiko/kex_gex.py new file mode 100644 index 00000000..c0455a1f --- /dev/null +++ b/contrib/site-packages/paramiko/kex_gex.py @@ -0,0 +1,243 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +Variant on L{KexGroup1 } where the prime "p" and +generator "g" are provided by the server. A bit more work is required on the +client side, and a B{lot} more on the server side. +""" + +from Crypto.Hash import SHA +from Crypto.Util import number + +from paramiko.common import * +from paramiko import util +from paramiko.message import Message +from paramiko.ssh_exception import SSHException + + +_MSG_KEXDH_GEX_REQUEST_OLD, _MSG_KEXDH_GEX_GROUP, _MSG_KEXDH_GEX_INIT, \ + _MSG_KEXDH_GEX_REPLY, _MSG_KEXDH_GEX_REQUEST = range(30, 35) + + +class KexGex (object): + + name = 'diffie-hellman-group-exchange-sha1' + min_bits = 1024 + max_bits = 8192 + preferred_bits = 2048 + + def __init__(self, transport): + self.transport = transport + self.p = None + self.q = None + self.g = None + self.x = None + self.e = None + self.f = None + self.old_style = False + + def start_kex(self, _test_old_style=False): + if self.transport.server_mode: + self.transport._expect_packet(_MSG_KEXDH_GEX_REQUEST, _MSG_KEXDH_GEX_REQUEST_OLD) + return + # request a bit range: we accept (min_bits) to (max_bits), but prefer + # (preferred_bits). according to the spec, we shouldn't pull the + # minimum up above 1024. + m = Message() + if _test_old_style: + # only used for unit tests: we shouldn't ever send this + m.add_byte(chr(_MSG_KEXDH_GEX_REQUEST_OLD)) + m.add_int(self.preferred_bits) + self.old_style = True + else: + m.add_byte(chr(_MSG_KEXDH_GEX_REQUEST)) + m.add_int(self.min_bits) + m.add_int(self.preferred_bits) + m.add_int(self.max_bits) + self.transport._send_message(m) + self.transport._expect_packet(_MSG_KEXDH_GEX_GROUP) + + def parse_next(self, ptype, m): + if ptype == _MSG_KEXDH_GEX_REQUEST: + return self._parse_kexdh_gex_request(m) + elif ptype == _MSG_KEXDH_GEX_GROUP: + return self._parse_kexdh_gex_group(m) + elif ptype == _MSG_KEXDH_GEX_INIT: + return self._parse_kexdh_gex_init(m) + elif ptype == _MSG_KEXDH_GEX_REPLY: + return self._parse_kexdh_gex_reply(m) + elif ptype == _MSG_KEXDH_GEX_REQUEST_OLD: + return self._parse_kexdh_gex_request_old(m) + raise SSHException('KexGex asked to handle packet type %d' % ptype) + + + ### internals... + + + def _generate_x(self): + # generate an "x" (1 < x < (p-1)/2). + q = (self.p - 1) // 2 + qnorm = util.deflate_long(q, 0) + qhbyte = ord(qnorm[0]) + bytes = len(qnorm) + qmask = 0xff + while not (qhbyte & 0x80): + qhbyte <<= 1 + qmask >>= 1 + while True: + x_bytes = self.transport.rng.read(bytes) + x_bytes = chr(ord(x_bytes[0]) & qmask) + x_bytes[1:] + x = util.inflate_long(x_bytes, 1) + if (x > 1) and (x < q): + break + self.x = x + + def _parse_kexdh_gex_request(self, m): + minbits = m.get_int() + preferredbits = m.get_int() + maxbits = m.get_int() + # smoosh the user's preferred size into our own limits + if preferredbits > self.max_bits: + preferredbits = self.max_bits + if preferredbits < self.min_bits: + preferredbits = self.min_bits + # fix min/max if they're inconsistent. technically, we could just pout + # and hang up, but there's no harm in giving them the benefit of the + # doubt and just picking a bitsize for them. + if minbits > preferredbits: + minbits = preferredbits + if maxbits < preferredbits: + maxbits = preferredbits + # now save a copy + self.min_bits = minbits + self.preferred_bits = preferredbits + self.max_bits = maxbits + # generate prime + pack = self.transport._get_modulus_pack() + if pack is None: + raise SSHException('Can\'t do server-side gex with no modulus pack') + self.transport._log(DEBUG, 'Picking p (%d <= %d <= %d bits)' % (minbits, preferredbits, maxbits)) + self.g, self.p = pack.get_modulus(minbits, preferredbits, maxbits) + m = Message() + m.add_byte(chr(_MSG_KEXDH_GEX_GROUP)) + m.add_mpint(self.p) + m.add_mpint(self.g) + self.transport._send_message(m) + self.transport._expect_packet(_MSG_KEXDH_GEX_INIT) + + def _parse_kexdh_gex_request_old(self, m): + # same as above, but without min_bits or max_bits (used by older clients like putty) + self.preferred_bits = m.get_int() + # smoosh the user's preferred size into our own limits + if self.preferred_bits > self.max_bits: + self.preferred_bits = self.max_bits + if self.preferred_bits < self.min_bits: + self.preferred_bits = self.min_bits + # generate prime + pack = self.transport._get_modulus_pack() + if pack is None: + raise SSHException('Can\'t do server-side gex with no modulus pack') + self.transport._log(DEBUG, 'Picking p (~ %d bits)' % (self.preferred_bits,)) + self.g, self.p = pack.get_modulus(self.min_bits, self.preferred_bits, self.max_bits) + m = Message() + m.add_byte(chr(_MSG_KEXDH_GEX_GROUP)) + m.add_mpint(self.p) + m.add_mpint(self.g) + self.transport._send_message(m) + self.transport._expect_packet(_MSG_KEXDH_GEX_INIT) + self.old_style = True + + def _parse_kexdh_gex_group(self, m): + self.p = m.get_mpint() + self.g = m.get_mpint() + # reject if p's bit length < 1024 or > 8192 + bitlen = util.bit_length(self.p) + if (bitlen < 1024) or (bitlen > 8192): + raise SSHException('Server-generated gex p (don\'t ask) is out of range (%d bits)' % bitlen) + self.transport._log(DEBUG, 'Got server p (%d bits)' % bitlen) + self._generate_x() + # now compute e = g^x mod p + self.e = pow(self.g, self.x, self.p) + m = Message() + m.add_byte(chr(_MSG_KEXDH_GEX_INIT)) + m.add_mpint(self.e) + self.transport._send_message(m) + self.transport._expect_packet(_MSG_KEXDH_GEX_REPLY) + + def _parse_kexdh_gex_init(self, m): + self.e = m.get_mpint() + if (self.e < 1) or (self.e > self.p - 1): + raise SSHException('Client kex "e" is out of range') + self._generate_x() + self.f = pow(self.g, self.x, self.p) + K = pow(self.e, self.x, self.p) + key = str(self.transport.get_server_key()) + # okay, build up the hash H of (V_C || V_S || I_C || I_S || K_S || min || n || max || p || g || e || f || K) + hm = Message() + hm.add(self.transport.remote_version, self.transport.local_version, + self.transport.remote_kex_init, self.transport.local_kex_init, + key) + if not self.old_style: + hm.add_int(self.min_bits) + hm.add_int(self.preferred_bits) + if not self.old_style: + hm.add_int(self.max_bits) + hm.add_mpint(self.p) + hm.add_mpint(self.g) + hm.add_mpint(self.e) + hm.add_mpint(self.f) + hm.add_mpint(K) + H = SHA.new(str(hm)).digest() + self.transport._set_K_H(K, H) + # sign it + sig = self.transport.get_server_key().sign_ssh_data(self.transport.rng, H) + # send reply + m = Message() + m.add_byte(chr(_MSG_KEXDH_GEX_REPLY)) + m.add_string(key) + m.add_mpint(self.f) + m.add_string(str(sig)) + self.transport._send_message(m) + self.transport._activate_outbound() + + def _parse_kexdh_gex_reply(self, m): + host_key = m.get_string() + self.f = m.get_mpint() + sig = m.get_string() + if (self.f < 1) or (self.f > self.p - 1): + raise SSHException('Server kex "f" is out of range') + K = pow(self.f, self.x, self.p) + # okay, build up the hash H of (V_C || V_S || I_C || I_S || K_S || min || n || max || p || g || e || f || K) + hm = Message() + hm.add(self.transport.local_version, self.transport.remote_version, + self.transport.local_kex_init, self.transport.remote_kex_init, + host_key) + if not self.old_style: + hm.add_int(self.min_bits) + hm.add_int(self.preferred_bits) + if not self.old_style: + hm.add_int(self.max_bits) + hm.add_mpint(self.p) + hm.add_mpint(self.g) + hm.add_mpint(self.e) + hm.add_mpint(self.f) + hm.add_mpint(K) + self.transport._set_K_H(K, SHA.new(str(hm)).digest()) + self.transport._verify_key(host_key, sig) + self.transport._activate_outbound() diff --git a/contrib/site-packages/paramiko/kex_group1.py b/contrib/site-packages/paramiko/kex_group1.py new file mode 100644 index 00000000..6e89b6dc --- /dev/null +++ b/contrib/site-packages/paramiko/kex_group1.py @@ -0,0 +1,135 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +Standard SSH key exchange ("kex" if you wanna sound cool). Diffie-Hellman of +1024 bit key halves, using a known "p" prime and "g" generator. +""" + +from Crypto.Hash import SHA + +from paramiko.common import * +from paramiko import util +from paramiko.message import Message +from paramiko.ssh_exception import SSHException + + +_MSG_KEXDH_INIT, _MSG_KEXDH_REPLY = range(30, 32) + +# draft-ietf-secsh-transport-09.txt, page 17 +P = 0xFFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E088A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE649286651ECE65381FFFFFFFFFFFFFFFFL +G = 2 + + +class KexGroup1(object): + + name = 'diffie-hellman-group1-sha1' + + def __init__(self, transport): + self.transport = transport + self.x = 0L + self.e = 0L + self.f = 0L + + def start_kex(self): + self._generate_x() + if self.transport.server_mode: + # compute f = g^x mod p, but don't send it yet + self.f = pow(G, self.x, P) + self.transport._expect_packet(_MSG_KEXDH_INIT) + return + # compute e = g^x mod p (where g=2), and send it + self.e = pow(G, self.x, P) + m = Message() + m.add_byte(chr(_MSG_KEXDH_INIT)) + m.add_mpint(self.e) + self.transport._send_message(m) + self.transport._expect_packet(_MSG_KEXDH_REPLY) + + def parse_next(self, ptype, m): + if self.transport.server_mode and (ptype == _MSG_KEXDH_INIT): + return self._parse_kexdh_init(m) + elif not self.transport.server_mode and (ptype == _MSG_KEXDH_REPLY): + return self._parse_kexdh_reply(m) + raise SSHException('KexGroup1 asked to handle packet type %d' % ptype) + + + ### internals... + + + def _generate_x(self): + # generate an "x" (1 < x < q), where q is (p-1)/2. + # p is a 128-byte (1024-bit) number, where the first 64 bits are 1. + # therefore q can be approximated as a 2^1023. we drop the subset of + # potential x where the first 63 bits are 1, because some of those will be + # larger than q (but this is a tiny tiny subset of potential x). + while 1: + x_bytes = self.transport.rng.read(128) + x_bytes = chr(ord(x_bytes[0]) & 0x7f) + x_bytes[1:] + if (x_bytes[:8] != '\x7F\xFF\xFF\xFF\xFF\xFF\xFF\xFF') and \ + (x_bytes[:8] != '\x00\x00\x00\x00\x00\x00\x00\x00'): + break + self.x = util.inflate_long(x_bytes) + + def _parse_kexdh_reply(self, m): + # client mode + host_key = m.get_string() + self.f = m.get_mpint() + if (self.f < 1) or (self.f > P - 1): + raise SSHException('Server kex "f" is out of range') + sig = m.get_string() + K = pow(self.f, self.x, P) + # okay, build up the hash H of (V_C || V_S || I_C || I_S || K_S || e || f || K) + hm = Message() + hm.add(self.transport.local_version, self.transport.remote_version, + self.transport.local_kex_init, self.transport.remote_kex_init) + hm.add_string(host_key) + hm.add_mpint(self.e) + hm.add_mpint(self.f) + hm.add_mpint(K) + self.transport._set_K_H(K, SHA.new(str(hm)).digest()) + self.transport._verify_key(host_key, sig) + self.transport._activate_outbound() + + def _parse_kexdh_init(self, m): + # server mode + self.e = m.get_mpint() + if (self.e < 1) or (self.e > P - 1): + raise SSHException('Client kex "e" is out of range') + K = pow(self.e, self.x, P) + key = str(self.transport.get_server_key()) + # okay, build up the hash H of (V_C || V_S || I_C || I_S || K_S || e || f || K) + hm = Message() + hm.add(self.transport.remote_version, self.transport.local_version, + self.transport.remote_kex_init, self.transport.local_kex_init) + hm.add_string(key) + hm.add_mpint(self.e) + hm.add_mpint(self.f) + hm.add_mpint(K) + H = SHA.new(str(hm)).digest() + self.transport._set_K_H(K, H) + # sign it + sig = self.transport.get_server_key().sign_ssh_data(self.transport.rng, H) + # send reply + m = Message() + m.add_byte(chr(_MSG_KEXDH_REPLY)) + m.add_string(key) + m.add_mpint(self.f) + m.add_string(str(sig)) + self.transport._send_message(m) + self.transport._activate_outbound() diff --git a/contrib/site-packages/paramiko/logging22.py b/contrib/site-packages/paramiko/logging22.py new file mode 100644 index 00000000..e68c52cb --- /dev/null +++ b/contrib/site-packages/paramiko/logging22.py @@ -0,0 +1,66 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +Stub out logging on python < 2.3. +""" + + +DEBUG = 10 +INFO = 20 +WARNING = 30 +ERROR = 40 +CRITICAL = 50 + + +def getLogger(name): + return _logger + + +class logger (object): + def __init__(self): + self.handlers = [ ] + self.level = ERROR + + def setLevel(self, level): + self.level = level + + def addHandler(self, h): + self.handlers.append(h) + + def addFilter(self, filter): + pass + + def log(self, level, text): + if level >= self.level: + for h in self.handlers: + h.f.write(text + '\n') + h.f.flush() + +class StreamHandler (object): + def __init__(self, f): + self.f = f + + def setFormatter(self, f): + pass + +class Formatter (object): + def __init__(self, x, y): + pass + +_logger = logger() diff --git a/contrib/site-packages/paramiko/message.py b/contrib/site-packages/paramiko/message.py new file mode 100644 index 00000000..c0e8692b --- /dev/null +++ b/contrib/site-packages/paramiko/message.py @@ -0,0 +1,302 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +Implementation of an SSH2 "message". +""" + +import struct +import cStringIO + +from paramiko import util + + +class Message (object): + """ + An SSH2 I{Message} is a stream of bytes that encodes some combination of + strings, integers, bools, and infinite-precision integers (known in python + as I{long}s). This class builds or breaks down such a byte stream. + + Normally you don't need to deal with anything this low-level, but it's + exposed for people implementing custom extensions, or features that + paramiko doesn't support yet. + """ + + def __init__(self, content=None): + """ + Create a new SSH2 Message. + + @param content: the byte stream to use as the Message content (passed + in only when decomposing a Message). + @type content: string + """ + if content != None: + self.packet = cStringIO.StringIO(content) + else: + self.packet = cStringIO.StringIO() + + def __str__(self): + """ + Return the byte stream content of this Message, as a string. + + @return: the contents of this Message. + @rtype: string + """ + return self.packet.getvalue() + + def __repr__(self): + """ + Returns a string representation of this object, for debugging. + + @rtype: string + """ + return 'paramiko.Message(' + repr(self.packet.getvalue()) + ')' + + def rewind(self): + """ + Rewind the message to the beginning as if no items had been parsed + out of it yet. + """ + self.packet.seek(0) + + def get_remainder(self): + """ + Return the bytes of this Message that haven't already been parsed and + returned. + + @return: a string of the bytes not parsed yet. + @rtype: string + """ + position = self.packet.tell() + remainder = self.packet.read() + self.packet.seek(position) + return remainder + + def get_so_far(self): + """ + Returns the bytes of this Message that have been parsed and returned. + The string passed into a Message's constructor can be regenerated by + concatenating C{get_so_far} and L{get_remainder}. + + @return: a string of the bytes parsed so far. + @rtype: string + """ + position = self.packet.tell() + self.rewind() + return self.packet.read(position) + + def get_bytes(self, n): + """ + Return the next C{n} bytes of the Message, without decomposing into + an int, string, etc. Just the raw bytes are returned. + + @return: a string of the next C{n} bytes of the Message, or a string + of C{n} zero bytes, if there aren't C{n} bytes remaining. + @rtype: string + """ + b = self.packet.read(n) + max_pad_size = 1<<20 # Limit padding to 1 MB + if len(b) < n and n < max_pad_size: + return b + '\x00' * (n - len(b)) + return b + + def get_byte(self): + """ + Return the next byte of the Message, without decomposing it. This + is equivalent to L{get_bytes(1)}. + + @return: the next byte of the Message, or C{'\000'} if there aren't + any bytes remaining. + @rtype: string + """ + return self.get_bytes(1) + + def get_boolean(self): + """ + Fetch a boolean from the stream. + + @return: C{True} or C{False} (from the Message). + @rtype: bool + """ + b = self.get_bytes(1) + return b != '\x00' + + def get_int(self): + """ + Fetch an int from the stream. + + @return: a 32-bit unsigned integer. + @rtype: int + """ + return struct.unpack('>I', self.get_bytes(4))[0] + + def get_int64(self): + """ + Fetch a 64-bit int from the stream. + + @return: a 64-bit unsigned integer. + @rtype: long + """ + return struct.unpack('>Q', self.get_bytes(8))[0] + + def get_mpint(self): + """ + Fetch a long int (mpint) from the stream. + + @return: an arbitrary-length integer. + @rtype: long + """ + return util.inflate_long(self.get_string()) + + def get_string(self): + """ + Fetch a string from the stream. This could be a byte string and may + contain unprintable characters. (It's not unheard of for a string to + contain another byte-stream Message.) + + @return: a string. + @rtype: string + """ + return self.get_bytes(self.get_int()) + + def get_list(self): + """ + Fetch a list of strings from the stream. These are trivially encoded + as comma-separated values in a string. + + @return: a list of strings. + @rtype: list of strings + """ + return self.get_string().split(',') + + def add_bytes(self, b): + """ + Write bytes to the stream, without any formatting. + + @param b: bytes to add + @type b: str + """ + self.packet.write(b) + return self + + def add_byte(self, b): + """ + Write a single byte to the stream, without any formatting. + + @param b: byte to add + @type b: str + """ + self.packet.write(b) + return self + + def add_boolean(self, b): + """ + Add a boolean value to the stream. + + @param b: boolean value to add + @type b: bool + """ + if b: + self.add_byte('\x01') + else: + self.add_byte('\x00') + return self + + def add_int(self, n): + """ + Add an integer to the stream. + + @param n: integer to add + @type n: int + """ + self.packet.write(struct.pack('>I', n)) + return self + + def add_int64(self, n): + """ + Add a 64-bit int to the stream. + + @param n: long int to add + @type n: long + """ + self.packet.write(struct.pack('>Q', n)) + return self + + def add_mpint(self, z): + """ + Add a long int to the stream, encoded as an infinite-precision + integer. This method only works on positive numbers. + + @param z: long int to add + @type z: long + """ + self.add_string(util.deflate_long(z)) + return self + + def add_string(self, s): + """ + Add a string to the stream. + + @param s: string to add + @type s: str + """ + self.add_int(len(s)) + self.packet.write(s) + return self + + def add_list(self, l): + """ + Add a list of strings to the stream. They are encoded identically to + a single string of values separated by commas. (Yes, really, that's + how SSH2 does it.) + + @param l: list of strings to add + @type l: list(str) + """ + self.add_string(','.join(l)) + return self + + def _add(self, i): + if type(i) is str: + return self.add_string(i) + elif type(i) is int: + return self.add_int(i) + elif type(i) is long: + if i > 0xffffffffL: + return self.add_mpint(i) + else: + return self.add_int(i) + elif type(i) is bool: + return self.add_boolean(i) + elif type(i) is list: + return self.add_list(i) + else: + raise Exception('Unknown type') + + def add(self, *seq): + """ + Add a sequence of items to the stream. The values are encoded based + on their type: str, int, bool, list, or long. + + @param seq: the sequence of items + @type seq: sequence + + @bug: longs are encoded non-deterministically. Don't use this method. + """ + for item in seq: + self._add(item) diff --git a/contrib/site-packages/paramiko/packet.py b/contrib/site-packages/paramiko/packet.py new file mode 100644 index 00000000..6ab7363d --- /dev/null +++ b/contrib/site-packages/paramiko/packet.py @@ -0,0 +1,500 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +Packetizer. +""" + +import errno +import select +import socket +import struct +import threading +import time + +from paramiko.common import * +from paramiko import util +from paramiko.ssh_exception import SSHException, ProxyCommandFailure +from paramiko.message import Message + + +try: + from r_hmac import HMAC +except ImportError: + from Crypto.Hash.HMAC import HMAC + +def compute_hmac(key, message, digest_class): + return HMAC(key, message, digest_class).digest() + + +class NeedRekeyException (Exception): + pass + + +class Packetizer (object): + """ + Implementation of the base SSH packet protocol. + """ + + # READ the secsh RFC's before raising these values. if anything, + # they should probably be lower. + REKEY_PACKETS = pow(2, 29) + REKEY_BYTES = pow(2, 29) + + REKEY_PACKETS_OVERFLOW_MAX = pow(2,29) # Allow receiving this many packets after a re-key request before terminating + REKEY_BYTES_OVERFLOW_MAX = pow(2,29) # Allow receiving this many bytes after a re-key request before terminating + + def __init__(self, socket): + self.__socket = socket + self.__logger = None + self.__closed = False + self.__dump_packets = False + self.__need_rekey = False + self.__init_count = 0 + self.__remainder = '' + + # used for noticing when to re-key: + self.__sent_bytes = 0 + self.__sent_packets = 0 + self.__received_bytes = 0 + self.__received_packets = 0 + self.__received_bytes_overflow = 0 + self.__received_packets_overflow = 0 + + # current inbound/outbound ciphering: + self.__block_size_out = 8 + self.__block_size_in = 8 + self.__mac_size_out = 0 + self.__mac_size_in = 0 + self.__block_engine_out = None + self.__block_engine_in = None + self.__sdctr_out = False + self.__mac_engine_out = None + self.__mac_engine_in = None + self.__mac_key_out = '' + self.__mac_key_in = '' + self.__compress_engine_out = None + self.__compress_engine_in = None + self.__sequence_number_out = 0L + self.__sequence_number_in = 0L + + # lock around outbound writes (packet computation) + self.__write_lock = threading.RLock() + + # keepalives: + self.__keepalive_interval = 0 + self.__keepalive_last = time.time() + self.__keepalive_callback = None + + def set_log(self, log): + """ + Set the python log object to use for logging. + """ + self.__logger = log + + def set_outbound_cipher(self, block_engine, block_size, mac_engine, mac_size, mac_key, sdctr=False): + """ + Switch outbound data cipher. + """ + self.__block_engine_out = block_engine + self.__sdctr_out = sdctr + self.__block_size_out = block_size + self.__mac_engine_out = mac_engine + self.__mac_size_out = mac_size + self.__mac_key_out = mac_key + self.__sent_bytes = 0 + self.__sent_packets = 0 + # wait until the reset happens in both directions before clearing rekey flag + self.__init_count |= 1 + if self.__init_count == 3: + self.__init_count = 0 + self.__need_rekey = False + + def set_inbound_cipher(self, block_engine, block_size, mac_engine, mac_size, mac_key): + """ + Switch inbound data cipher. + """ + self.__block_engine_in = block_engine + self.__block_size_in = block_size + self.__mac_engine_in = mac_engine + self.__mac_size_in = mac_size + self.__mac_key_in = mac_key + self.__received_bytes = 0 + self.__received_packets = 0 + self.__received_bytes_overflow = 0 + self.__received_packets_overflow = 0 + # wait until the reset happens in both directions before clearing rekey flag + self.__init_count |= 2 + if self.__init_count == 3: + self.__init_count = 0 + self.__need_rekey = False + + def set_outbound_compressor(self, compressor): + self.__compress_engine_out = compressor + + def set_inbound_compressor(self, compressor): + self.__compress_engine_in = compressor + + def close(self): + self.__closed = True + + def set_hexdump(self, hexdump): + self.__dump_packets = hexdump + + def get_hexdump(self): + return self.__dump_packets + + def get_mac_size_in(self): + return self.__mac_size_in + + def get_mac_size_out(self): + return self.__mac_size_out + + def need_rekey(self): + """ + Returns C{True} if a new set of keys needs to be negotiated. This + will be triggered during a packet read or write, so it should be + checked after every read or write, or at least after every few. + + @return: C{True} if a new set of keys needs to be negotiated + """ + return self.__need_rekey + + def set_keepalive(self, interval, callback): + """ + Turn on/off the callback keepalive. If C{interval} seconds pass with + no data read from or written to the socket, the callback will be + executed and the timer will be reset. + """ + self.__keepalive_interval = interval + self.__keepalive_callback = callback + self.__keepalive_last = time.time() + + def read_all(self, n, check_rekey=False): + """ + Read as close to N bytes as possible, blocking as long as necessary. + + @param n: number of bytes to read + @type n: int + @return: the data read + @rtype: str + @raise EOFError: if the socket was closed before all the bytes could + be read + """ + out = '' + # handle over-reading from reading the banner line + if len(self.__remainder) > 0: + out = self.__remainder[:n] + self.__remainder = self.__remainder[n:] + n -= len(out) + if PY22: + return self._py22_read_all(n, out) + while n > 0: + got_timeout = False + try: + x = self.__socket.recv(n) + if len(x) == 0: + raise EOFError() + out += x + n -= len(x) + except socket.timeout: + got_timeout = True + except socket.error, e: + # on Linux, sometimes instead of socket.timeout, we get + # EAGAIN. this is a bug in recent (> 2.6.9) kernels but + # we need to work around it. + if (type(e.args) is tuple) and (len(e.args) > 0) and (e.args[0] == errno.EAGAIN): + got_timeout = True + elif (type(e.args) is tuple) and (len(e.args) > 0) and (e.args[0] == errno.EINTR): + # syscall interrupted; try again + pass + elif self.__closed: + raise EOFError() + else: + raise + if got_timeout: + if self.__closed: + raise EOFError() + if check_rekey and (len(out) == 0) and self.__need_rekey: + raise NeedRekeyException() + self._check_keepalive() + return out + + def write_all(self, out): + self.__keepalive_last = time.time() + while len(out) > 0: + retry_write = False + try: + n = self.__socket.send(out) + except socket.timeout: + retry_write = True + except socket.error, e: + if (type(e.args) is tuple) and (len(e.args) > 0) and (e.args[0] == errno.EAGAIN): + retry_write = True + elif (type(e.args) is tuple) and (len(e.args) > 0) and (e.args[0] == errno.EINTR): + # syscall interrupted; try again + retry_write = True + else: + n = -1 + except ProxyCommandFailure: + raise # so it doesn't get swallowed by the below catchall + except Exception: + # could be: (32, 'Broken pipe') + n = -1 + if retry_write: + n = 0 + if self.__closed: + n = -1 + if n < 0: + raise EOFError() + if n == len(out): + break + out = out[n:] + return + + def readline(self, timeout): + """ + Read a line from the socket. We assume no data is pending after the + line, so it's okay to attempt large reads. + """ + buf = self.__remainder + while not '\n' in buf: + buf += self._read_timeout(timeout) + n = buf.index('\n') + self.__remainder = buf[n+1:] + buf = buf[:n] + if (len(buf) > 0) and (buf[-1] == '\r'): + buf = buf[:-1] + return buf + + def send_message(self, data): + """ + Write a block of data using the current cipher, as an SSH block. + """ + # encrypt this sucka + data = str(data) + cmd = ord(data[0]) + if cmd in MSG_NAMES: + cmd_name = MSG_NAMES[cmd] + else: + cmd_name = '$%x' % cmd + orig_len = len(data) + self.__write_lock.acquire() + try: + if self.__compress_engine_out is not None: + data = self.__compress_engine_out(data) + packet = self._build_packet(data) + if self.__dump_packets: + self._log(DEBUG, 'Write packet <%s>, length %d' % (cmd_name, orig_len)) + self._log(DEBUG, util.format_binary(packet, 'OUT: ')) + if self.__block_engine_out != None: + out = self.__block_engine_out.encrypt(packet) + else: + out = packet + # + mac + if self.__block_engine_out != None: + payload = struct.pack('>I', self.__sequence_number_out) + packet + out += compute_hmac(self.__mac_key_out, payload, self.__mac_engine_out)[:self.__mac_size_out] + self.__sequence_number_out = (self.__sequence_number_out + 1) & 0xffffffffL + self.write_all(out) + + self.__sent_bytes += len(out) + self.__sent_packets += 1 + if ((self.__sent_packets >= self.REKEY_PACKETS) or (self.__sent_bytes >= self.REKEY_BYTES)) \ + and not self.__need_rekey: + # only ask once for rekeying + self._log(DEBUG, 'Rekeying (hit %d packets, %d bytes sent)' % + (self.__sent_packets, self.__sent_bytes)) + self.__received_bytes_overflow = 0 + self.__received_packets_overflow = 0 + self._trigger_rekey() + finally: + self.__write_lock.release() + + def read_message(self): + """ + Only one thread should ever be in this function (no other locking is + done). + + @raise SSHException: if the packet is mangled + @raise NeedRekeyException: if the transport should rekey + """ + header = self.read_all(self.__block_size_in, check_rekey=True) + if self.__block_engine_in != None: + header = self.__block_engine_in.decrypt(header) + if self.__dump_packets: + self._log(DEBUG, util.format_binary(header, 'IN: ')); + packet_size = struct.unpack('>I', header[:4])[0] + # leftover contains decrypted bytes from the first block (after the length field) + leftover = header[4:] + if (packet_size - len(leftover)) % self.__block_size_in != 0: + raise SSHException('Invalid packet blocking') + buf = self.read_all(packet_size + self.__mac_size_in - len(leftover)) + packet = buf[:packet_size - len(leftover)] + post_packet = buf[packet_size - len(leftover):] + if self.__block_engine_in != None: + packet = self.__block_engine_in.decrypt(packet) + if self.__dump_packets: + self._log(DEBUG, util.format_binary(packet, 'IN: ')); + packet = leftover + packet + + if self.__mac_size_in > 0: + mac = post_packet[:self.__mac_size_in] + mac_payload = struct.pack('>II', self.__sequence_number_in, packet_size) + packet + my_mac = compute_hmac(self.__mac_key_in, mac_payload, self.__mac_engine_in)[:self.__mac_size_in] + if my_mac != mac: + raise SSHException('Mismatched MAC') + padding = ord(packet[0]) + payload = packet[1:packet_size - padding] + + if self.__dump_packets: + self._log(DEBUG, 'Got payload (%d bytes, %d padding)' % (packet_size, padding)) + + if self.__compress_engine_in is not None: + payload = self.__compress_engine_in(payload) + + msg = Message(payload[1:]) + msg.seqno = self.__sequence_number_in + self.__sequence_number_in = (self.__sequence_number_in + 1) & 0xffffffffL + + # check for rekey + raw_packet_size = packet_size + self.__mac_size_in + 4 + self.__received_bytes += raw_packet_size + self.__received_packets += 1 + if self.__need_rekey: + # we've asked to rekey -- give them some packets to comply before + # dropping the connection + self.__received_bytes_overflow += raw_packet_size + self.__received_packets_overflow += 1 + if (self.__received_packets_overflow >= self.REKEY_PACKETS_OVERFLOW_MAX) or \ + (self.__received_bytes_overflow >= self.REKEY_BYTES_OVERFLOW_MAX): + raise SSHException('Remote transport is ignoring rekey requests') + elif (self.__received_packets >= self.REKEY_PACKETS) or \ + (self.__received_bytes >= self.REKEY_BYTES): + # only ask once for rekeying + self._log(DEBUG, 'Rekeying (hit %d packets, %d bytes received)' % + (self.__received_packets, self.__received_bytes)) + self.__received_bytes_overflow = 0 + self.__received_packets_overflow = 0 + self._trigger_rekey() + + cmd = ord(payload[0]) + if cmd in MSG_NAMES: + cmd_name = MSG_NAMES[cmd] + else: + cmd_name = '$%x' % cmd + if self.__dump_packets: + self._log(DEBUG, 'Read packet <%s>, length %d' % (cmd_name, len(payload))) + return cmd, msg + + + ########## protected + + + def _log(self, level, msg): + if self.__logger is None: + return + if issubclass(type(msg), list): + for m in msg: + self.__logger.log(level, m) + else: + self.__logger.log(level, msg) + + def _check_keepalive(self): + if (not self.__keepalive_interval) or (not self.__block_engine_out) or \ + self.__need_rekey: + # wait till we're encrypting, and not in the middle of rekeying + return + now = time.time() + if now > self.__keepalive_last + self.__keepalive_interval: + self.__keepalive_callback() + self.__keepalive_last = now + + def _py22_read_all(self, n, out): + while n > 0: + r, w, e = select.select([self.__socket], [], [], 0.1) + if self.__socket not in r: + if self.__closed: + raise EOFError() + self._check_keepalive() + else: + x = self.__socket.recv(n) + if len(x) == 0: + raise EOFError() + out += x + n -= len(x) + return out + + def _py22_read_timeout(self, timeout): + start = time.time() + while True: + r, w, e = select.select([self.__socket], [], [], 0.1) + if self.__socket in r: + x = self.__socket.recv(1) + if len(x) == 0: + raise EOFError() + break + if self.__closed: + raise EOFError() + now = time.time() + if now - start >= timeout: + raise socket.timeout() + return x + + def _read_timeout(self, timeout): + if PY22: + return self._py22_read_timeout(timeout) + start = time.time() + while True: + try: + x = self.__socket.recv(128) + if len(x) == 0: + raise EOFError() + break + except socket.timeout: + pass + except EnvironmentError, e: + if ((type(e.args) is tuple) and (len(e.args) > 0) and + (e.args[0] == errno.EINTR)): + pass + else: + raise + if self.__closed: + raise EOFError() + now = time.time() + if now - start >= timeout: + raise socket.timeout() + return x + + def _build_packet(self, payload): + # pad up at least 4 bytes, to nearest block-size (usually 8) + bsize = self.__block_size_out + padding = 3 + bsize - ((len(payload) + 8) % bsize) + packet = struct.pack('>IB', len(payload) + padding + 1, padding) + packet += payload + if self.__sdctr_out or self.__block_engine_out is None: + # cute trick i caught openssh doing: if we're not encrypting or SDCTR mode (RFC4344), + # don't waste random bytes for the padding + packet += (chr(0) * padding) + else: + packet += rng.read(padding) + return packet + + def _trigger_rekey(self): + # outside code should check for this flag + self.__need_rekey = True diff --git a/contrib/site-packages/paramiko/pipe.py b/contrib/site-packages/paramiko/pipe.py new file mode 100644 index 00000000..db43d549 --- /dev/null +++ b/contrib/site-packages/paramiko/pipe.py @@ -0,0 +1,147 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +Abstraction of a one-way pipe where the read end can be used in select(). +Normally this is trivial, but Windows makes it nearly impossible. + +The pipe acts like an Event, which can be set or cleared. When set, the pipe +will trigger as readable in select(). +""" + +import sys +import os +import socket + + +def make_pipe (): + if sys.platform[:3] != 'win': + p = PosixPipe() + else: + p = WindowsPipe() + return p + + +class PosixPipe (object): + def __init__ (self): + self._rfd, self._wfd = os.pipe() + self._set = False + self._forever = False + self._closed = False + + def close (self): + os.close(self._rfd) + os.close(self._wfd) + # used for unit tests: + self._closed = True + + def fileno (self): + return self._rfd + + def clear (self): + if not self._set or self._forever: + return + os.read(self._rfd, 1) + self._set = False + + def set (self): + if self._set or self._closed: + return + self._set = True + os.write(self._wfd, '*') + + def set_forever (self): + self._forever = True + self.set() + + +class WindowsPipe (object): + """ + On Windows, only an OS-level "WinSock" may be used in select(), but reads + and writes must be to the actual socket object. + """ + def __init__ (self): + serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + serv.bind(('127.0.0.1', 0)) + serv.listen(1) + + # need to save sockets in _rsock/_wsock so they don't get closed + self._rsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._rsock.connect(('127.0.0.1', serv.getsockname()[1])) + + self._wsock, addr = serv.accept() + serv.close() + self._set = False + self._forever = False + self._closed = False + + def close (self): + self._rsock.close() + self._wsock.close() + # used for unit tests: + self._closed = True + + def fileno (self): + return self._rsock.fileno() + + def clear (self): + if not self._set or self._forever: + return + self._rsock.recv(1) + self._set = False + + def set (self): + if self._set or self._closed: + return + self._set = True + self._wsock.send('*') + + def set_forever (self): + self._forever = True + self.set() + + +class OrPipe (object): + def __init__(self, pipe): + self._set = False + self._partner = None + self._pipe = pipe + + def set(self): + self._set = True + if not self._partner._set: + self._pipe.set() + + def clear(self): + self._set = False + if not self._partner._set: + self._pipe.clear() + + +def make_or_pipe(pipe): + """ + wraps a pipe into two pipe-like objects which are "or"d together to + affect the real pipe. if either returned pipe is set, the wrapped pipe + is set. when both are cleared, the wrapped pipe is cleared. + """ + p1 = OrPipe(pipe) + p2 = OrPipe(pipe) + p1._partner = p2 + p2._partner = p1 + return p1, p2 + diff --git a/contrib/site-packages/paramiko/pkey.py b/contrib/site-packages/paramiko/pkey.py new file mode 100644 index 00000000..b1199df8 --- /dev/null +++ b/contrib/site-packages/paramiko/pkey.py @@ -0,0 +1,381 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +Common API for all public keys. +""" + +import base64 +from binascii import hexlify, unhexlify +import os + +from Crypto.Hash import MD5 +from Crypto.Cipher import DES3, AES + +from paramiko.common import * +from paramiko import util +from paramiko.message import Message +from paramiko.ssh_exception import SSHException, PasswordRequiredException + + +class PKey (object): + """ + Base class for public keys. + """ + + # known encryption types for private key files: + _CIPHER_TABLE = { + 'AES-128-CBC': { 'cipher': AES, 'keysize': 16, 'blocksize': 16, 'mode': AES.MODE_CBC }, + 'DES-EDE3-CBC': { 'cipher': DES3, 'keysize': 24, 'blocksize': 8, 'mode': DES3.MODE_CBC }, + } + + + def __init__(self, msg=None, data=None): + """ + Create a new instance of this public key type. If C{msg} is given, + the key's public part(s) will be filled in from the message. If + C{data} is given, the key's public part(s) will be filled in from + the string. + + @param msg: an optional SSH L{Message} containing a public key of this + type. + @type msg: L{Message} + @param data: an optional string containing a public key of this type + @type data: str + + @raise SSHException: if a key cannot be created from the C{data} or + C{msg} given, or no key was passed in. + """ + pass + + def __str__(self): + """ + Return a string of an SSH L{Message} made up of the public part(s) of + this key. This string is suitable for passing to L{__init__} to + re-create the key object later. + + @return: string representation of an SSH key message. + @rtype: str + """ + return '' + + def __cmp__(self, other): + """ + Compare this key to another. Returns 0 if this key is equivalent to + the given key, or non-0 if they are different. Only the public parts + of the key are compared, so a public key will compare equal to its + corresponding private key. + + @param other: key to compare to. + @type other: L{PKey} + @return: 0 if the two keys are equivalent, non-0 otherwise. + @rtype: int + """ + hs = hash(self) + ho = hash(other) + if hs != ho: + return cmp(hs, ho) + return cmp(str(self), str(other)) + + def get_name(self): + """ + Return the name of this private key implementation. + + @return: name of this private key type, in SSH terminology (for + example, C{"ssh-rsa"}). + @rtype: str + """ + return '' + + def get_bits(self): + """ + Return the number of significant bits in this key. This is useful + for judging the relative security of a key. + + @return: bits in the key. + @rtype: int + """ + return 0 + + def can_sign(self): + """ + Return C{True} if this key has the private part necessary for signing + data. + + @return: C{True} if this is a private key. + @rtype: bool + """ + return False + + def get_fingerprint(self): + """ + Return an MD5 fingerprint of the public part of this key. Nothing + secret is revealed. + + @return: a 16-byte string (binary) of the MD5 fingerprint, in SSH + format. + @rtype: str + """ + return MD5.new(str(self)).digest() + + def get_base64(self): + """ + Return a base64 string containing the public part of this key. Nothing + secret is revealed. This format is compatible with that used to store + public key files or recognized host keys. + + @return: a base64 string containing the public part of the key. + @rtype: str + """ + return base64.encodestring(str(self)).replace('\n', '') + + def sign_ssh_data(self, rng, data): + """ + Sign a blob of data with this private key, and return a L{Message} + representing an SSH signature message. + + @param rng: a secure random number generator. + @type rng: L{Crypto.Util.rng.RandomPool} + @param data: the data to sign. + @type data: str + @return: an SSH signature message. + @rtype: L{Message} + """ + return '' + + def verify_ssh_sig(self, data, msg): + """ + Given a blob of data, and an SSH message representing a signature of + that data, verify that it was signed with this key. + + @param data: the data that was signed. + @type data: str + @param msg: an SSH signature message + @type msg: L{Message} + @return: C{True} if the signature verifies correctly; C{False} + otherwise. + @rtype: boolean + """ + return False + + def from_private_key_file(cls, filename, password=None): + """ + Create a key object by reading a private key file. If the private + key is encrypted and C{password} is not C{None}, the given password + will be used to decrypt the key (otherwise L{PasswordRequiredException} + is thrown). Through the magic of python, this factory method will + exist in all subclasses of PKey (such as L{RSAKey} or L{DSSKey}), but + is useless on the abstract PKey class. + + @param filename: name of the file to read + @type filename: str + @param password: an optional password to use to decrypt the key file, + if it's encrypted + @type password: str + @return: a new key object based on the given private key + @rtype: L{PKey} + + @raise IOError: if there was an error reading the file + @raise PasswordRequiredException: if the private key file is + encrypted, and C{password} is C{None} + @raise SSHException: if the key file is invalid + """ + key = cls(filename=filename, password=password) + return key + from_private_key_file = classmethod(from_private_key_file) + + def from_private_key(cls, file_obj, password=None): + """ + Create a key object by reading a private key from a file (or file-like) + object. If the private key is encrypted and C{password} is not C{None}, + the given password will be used to decrypt the key (otherwise + L{PasswordRequiredException} is thrown). + + @param file_obj: the file to read from + @type file_obj: file + @param password: an optional password to use to decrypt the key, if it's + encrypted + @type password: str + @return: a new key object based on the given private key + @rtype: L{PKey} + + @raise IOError: if there was an error reading the key + @raise PasswordRequiredException: if the private key file is encrypted, + and C{password} is C{None} + @raise SSHException: if the key file is invalid + """ + key = cls(file_obj=file_obj, password=password) + return key + from_private_key = classmethod(from_private_key) + + def write_private_key_file(self, filename, password=None): + """ + Write private key contents into a file. If the password is not + C{None}, the key is encrypted before writing. + + @param filename: name of the file to write + @type filename: str + @param password: an optional password to use to encrypt the key file + @type password: str + + @raise IOError: if there was an error writing the file + @raise SSHException: if the key is invalid + """ + raise Exception('Not implemented in PKey') + + def write_private_key(self, file_obj, password=None): + """ + Write private key contents into a file (or file-like) object. If the + password is not C{None}, the key is encrypted before writing. + + @param file_obj: the file object to write into + @type file_obj: file + @param password: an optional password to use to encrypt the key + @type password: str + + @raise IOError: if there was an error writing to the file + @raise SSHException: if the key is invalid + """ + raise Exception('Not implemented in PKey') + + def _read_private_key_file(self, tag, filename, password=None): + """ + Read an SSH2-format private key file, looking for a string of the type + C{"BEGIN xxx PRIVATE KEY"} for some C{xxx}, base64-decode the text we + find, and return it as a string. If the private key is encrypted and + C{password} is not C{None}, the given password will be used to decrypt + the key (otherwise L{PasswordRequiredException} is thrown). + + @param tag: C{"RSA"} or C{"DSA"}, the tag used to mark the data block. + @type tag: str + @param filename: name of the file to read. + @type filename: str + @param password: an optional password to use to decrypt the key file, + if it's encrypted. + @type password: str + @return: data blob that makes up the private key. + @rtype: str + + @raise IOError: if there was an error reading the file. + @raise PasswordRequiredException: if the private key file is + encrypted, and C{password} is C{None}. + @raise SSHException: if the key file is invalid. + """ + f = open(filename, 'r') + data = self._read_private_key(tag, f, password) + f.close() + return data + + def _read_private_key(self, tag, f, password=None): + lines = f.readlines() + start = 0 + while (start < len(lines)) and (lines[start].strip() != '-----BEGIN ' + tag + ' PRIVATE KEY-----'): + start += 1 + if start >= len(lines): + raise SSHException('not a valid ' + tag + ' private key file') + # parse any headers first + headers = {} + start += 1 + while start < len(lines): + l = lines[start].split(': ') + if len(l) == 1: + break + headers[l[0].lower()] = l[1].strip() + start += 1 + # find end + end = start + while (lines[end].strip() != '-----END ' + tag + ' PRIVATE KEY-----') and (end < len(lines)): + end += 1 + # if we trudged to the end of the file, just try to cope. + try: + data = base64.decodestring(''.join(lines[start:end])) + except base64.binascii.Error, e: + raise SSHException('base64 decoding error: ' + str(e)) + if 'proc-type' not in headers: + # unencryped: done + return data + # encrypted keyfile: will need a password + if headers['proc-type'] != '4,ENCRYPTED': + raise SSHException('Unknown private key structure "%s"' % headers['proc-type']) + try: + encryption_type, saltstr = headers['dek-info'].split(',') + except: + raise SSHException('Can\'t parse DEK-info in private key file') + if encryption_type not in self._CIPHER_TABLE: + raise SSHException('Unknown private key cipher "%s"' % encryption_type) + # if no password was passed in, raise an exception pointing out that we need one + if password is None: + raise PasswordRequiredException('Private key file is encrypted') + cipher = self._CIPHER_TABLE[encryption_type]['cipher'] + keysize = self._CIPHER_TABLE[encryption_type]['keysize'] + mode = self._CIPHER_TABLE[encryption_type]['mode'] + salt = unhexlify(saltstr) + key = util.generate_key_bytes(MD5, salt, password, keysize) + return cipher.new(key, mode, salt).decrypt(data) + + def _write_private_key_file(self, tag, filename, data, password=None): + """ + Write an SSH2-format private key file in a form that can be read by + paramiko or openssh. If no password is given, the key is written in + a trivially-encoded format (base64) which is completely insecure. If + a password is given, DES-EDE3-CBC is used. + + @param tag: C{"RSA"} or C{"DSA"}, the tag used to mark the data block. + @type tag: str + @param filename: name of the file to write. + @type filename: str + @param data: data blob that makes up the private key. + @type data: str + @param password: an optional password to use to encrypt the file. + @type password: str + + @raise IOError: if there was an error writing the file. + """ + f = open(filename, 'w', 0600) + # grrr... the mode doesn't always take hold + os.chmod(filename, 0600) + self._write_private_key(tag, f, data, password) + f.close() + + def _write_private_key(self, tag, f, data, password=None): + f.write('-----BEGIN %s PRIVATE KEY-----\n' % tag) + if password is not None: + # since we only support one cipher here, use it + cipher_name = self._CIPHER_TABLE.keys()[0] + cipher = self._CIPHER_TABLE[cipher_name]['cipher'] + keysize = self._CIPHER_TABLE[cipher_name]['keysize'] + blocksize = self._CIPHER_TABLE[cipher_name]['blocksize'] + mode = self._CIPHER_TABLE[cipher_name]['mode'] + salt = rng.read(8) + key = util.generate_key_bytes(MD5, salt, password, keysize) + if len(data) % blocksize != 0: + n = blocksize - len(data) % blocksize + #data += rng.read(n) + # that would make more sense ^, but it confuses openssh. + data += '\0' * n + data = cipher.new(key, mode, salt).encrypt(data) + f.write('Proc-Type: 4,ENCRYPTED\n') + f.write('DEK-Info: %s,%s\n' % (cipher_name, hexlify(salt).upper())) + f.write('\n') + s = base64.encodestring(data) + # re-wrap to 64-char lines + s = ''.join(s.split('\n')) + s = '\n'.join([s[i : i+64] for i in range(0, len(s), 64)]) + f.write(s) + f.write('\n') + f.write('-----END %s PRIVATE KEY-----\n' % tag) diff --git a/contrib/site-packages/paramiko/primes.py b/contrib/site-packages/paramiko/primes.py new file mode 100644 index 00000000..9419cd6b --- /dev/null +++ b/contrib/site-packages/paramiko/primes.py @@ -0,0 +1,151 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +Utility functions for dealing with primes. +""" + +from Crypto.Util import number + +from paramiko import util +from paramiko.ssh_exception import SSHException + + +def _generate_prime(bits, rng): + "primtive attempt at prime generation" + hbyte_mask = pow(2, bits % 8) - 1 + while True: + # loop catches the case where we increment n into a higher bit-range + x = rng.read((bits+7) // 8) + if hbyte_mask > 0: + x = chr(ord(x[0]) & hbyte_mask) + x[1:] + n = util.inflate_long(x, 1) + n |= 1 + n |= (1 << (bits - 1)) + while not number.isPrime(n): + n += 2 + if util.bit_length(n) == bits: + break + return n + +def _roll_random(rng, n): + "returns a random # from 0 to N-1" + bits = util.bit_length(n-1) + bytes = (bits + 7) // 8 + hbyte_mask = pow(2, bits % 8) - 1 + + # so here's the plan: + # we fetch as many random bits as we'd need to fit N-1, and if the + # generated number is >= N, we try again. in the worst case (N-1 is a + # power of 2), we have slightly better than 50% odds of getting one that + # fits, so i can't guarantee that this loop will ever finish, but the odds + # of it looping forever should be infinitesimal. + while True: + x = rng.read(bytes) + if hbyte_mask > 0: + x = chr(ord(x[0]) & hbyte_mask) + x[1:] + num = util.inflate_long(x, 1) + if num < n: + break + return num + + +class ModulusPack (object): + """ + convenience object for holding the contents of the /etc/ssh/moduli file, + on systems that have such a file. + """ + + def __init__(self, rpool): + # pack is a hash of: bits -> [ (generator, modulus) ... ] + self.pack = {} + self.discarded = [] + self.rng = rpool + + def _parse_modulus(self, line): + timestamp, mod_type, tests, tries, size, generator, modulus = line.split() + mod_type = int(mod_type) + tests = int(tests) + tries = int(tries) + size = int(size) + generator = int(generator) + modulus = long(modulus, 16) + + # weed out primes that aren't at least: + # type 2 (meets basic structural requirements) + # test 4 (more than just a small-prime sieve) + # tries < 100 if test & 4 (at least 100 tries of miller-rabin) + if (mod_type < 2) or (tests < 4) or ((tests & 4) and (tests < 8) and (tries < 100)): + self.discarded.append((modulus, 'does not meet basic requirements')) + return + if generator == 0: + generator = 2 + + # there's a bug in the ssh "moduli" file (yeah, i know: shock! dismay! + # call cnn!) where it understates the bit lengths of these primes by 1. + # this is okay. + bl = util.bit_length(modulus) + if (bl != size) and (bl != size + 1): + self.discarded.append((modulus, 'incorrectly reported bit length %d' % size)) + return + if bl not in self.pack: + self.pack[bl] = [] + self.pack[bl].append((generator, modulus)) + + def read_file(self, filename): + """ + @raise IOError: passed from any file operations that fail. + """ + self.pack = {} + f = open(filename, 'r') + for line in f: + line = line.strip() + if (len(line) == 0) or (line[0] == '#'): + continue + try: + self._parse_modulus(line) + except: + continue + f.close() + + def get_modulus(self, min, prefer, max): + bitsizes = self.pack.keys() + bitsizes.sort() + if len(bitsizes) == 0: + raise SSHException('no moduli available') + good = -1 + # find nearest bitsize >= preferred + for b in bitsizes: + if (b >= prefer) and (b < max) and ((b < good) or (good == -1)): + good = b + # if that failed, find greatest bitsize >= min + if good == -1: + for b in bitsizes: + if (b >= min) and (b < max) and (b > good): + good = b + if good == -1: + # their entire (min, max) range has no intersection with our range. + # if their range is below ours, pick the smallest. otherwise pick + # the largest. it'll be out of their range requirement either way, + # but we'll be sending them the closest one we have. + good = bitsizes[0] + if min > good: + good = bitsizes[-1] + # now pick a random modulus of this bitsize + n = _roll_random(self.rng, len(self.pack[good])) + return self.pack[good][n] diff --git a/contrib/site-packages/paramiko/proxy.py b/contrib/site-packages/paramiko/proxy.py new file mode 100644 index 00000000..218b76e2 --- /dev/null +++ b/contrib/site-packages/paramiko/proxy.py @@ -0,0 +1,91 @@ +# Copyright (C) 2012 Yipit, Inc +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distrubuted in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +L{ProxyCommand}. +""" + +import os +from shlex import split as shlsplit +import signal +from subprocess import Popen, PIPE + +from paramiko.ssh_exception import ProxyCommandFailure + + +class ProxyCommand(object): + """ + Wraps a subprocess running ProxyCommand-driven programs. + + This class implements a the socket-like interface needed by the + L{Transport} and L{Packetizer} classes. Using this class instead of a + regular socket makes it possible to talk with a Popen'd command that will + proxy traffic between the client and a server hosted in another machine. + """ + def __init__(self, command_line): + """ + Create a new CommandProxy instance. The instance created by this + class can be passed as an argument to the L{Transport} class. + + @param command_line: the command that should be executed and + used as the proxy. + @type command_line: str + """ + self.cmd = shlsplit(command_line) + self.process = Popen(self.cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) + + def send(self, content): + """ + Write the content received from the SSH client to the standard + input of the forked command. + + @param content: string to be sent to the forked command + @type content: str + """ + try: + self.process.stdin.write(content) + except IOError, e: + # There was a problem with the child process. It probably + # died and we can't proceed. The best option here is to + # raise an exception informing the user that the informed + # ProxyCommand is not working. + raise BadProxyCommand(' '.join(self.cmd), e.strerror) + return len(content) + + def recv(self, size): + """ + Read from the standard output of the forked program. + + @param size: how many chars should be read + @type size: int + + @return: the length of the read content + @rtype: int + """ + try: + return os.read(self.process.stdout.fileno(), size) + except IOError, e: + raise BadProxyCommand(' '.join(self.cmd), e.strerror) + + def close(self): + os.kill(self.process.pid, signal.SIGTERM) + + def settimeout(self, timeout): + # Timeouts are meaningless for this implementation, but are part of the + # spec, so must be present. + pass diff --git a/contrib/site-packages/paramiko/resource.py b/contrib/site-packages/paramiko/resource.py new file mode 100644 index 00000000..6ef86d8c --- /dev/null +++ b/contrib/site-packages/paramiko/resource.py @@ -0,0 +1,72 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +Resource manager. +""" + +import weakref + + +class ResourceManager (object): + """ + A registry of objects and resources that should be closed when those + objects are deleted. + + This is meant to be a safer alternative to python's C{__del__} method, + which can cause reference cycles to never be collected. Objects registered + with the ResourceManager can be collected but still free resources when + they die. + + Resources are registered using L{register}, and when an object is garbage + collected, each registered resource is closed by having its C{close()} + method called. Multiple resources may be registered per object, but a + resource will only be closed once, even if multiple objects register it. + (The last object to register it wins.) + """ + + def __init__(self): + self._table = {} + + def register(self, obj, resource): + """ + Register a resource to be closed with an object is collected. + + When the given C{obj} is garbage-collected by the python interpreter, + the C{resource} will be closed by having its C{close()} method called. + Any exceptions are ignored. + + @param obj: the object to track + @type obj: object + @param resource: the resource to close when the object is collected + @type resource: object + """ + def callback(ref): + try: + resource.close() + except: + pass + del self._table[id(resource)] + + # keep the weakref in a table so it sticks around long enough to get + # its callback called. :) + self._table[id(resource)] = weakref.ref(obj, callback) + + +# singleton +ResourceManager = ResourceManager() diff --git a/contrib/site-packages/paramiko/rsakey.py b/contrib/site-packages/paramiko/rsakey.py new file mode 100644 index 00000000..c7500f85 --- /dev/null +++ b/contrib/site-packages/paramiko/rsakey.py @@ -0,0 +1,185 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +L{RSAKey} +""" + +from Crypto.PublicKey import RSA +from Crypto.Hash import SHA, MD5 +from Crypto.Cipher import DES3 + +from paramiko.common import * +from paramiko import util +from paramiko.message import Message +from paramiko.ber import BER, BERException +from paramiko.pkey import PKey +from paramiko.ssh_exception import SSHException + + +class RSAKey (PKey): + """ + Representation of an RSA key which can be used to sign and verify SSH2 + data. + """ + + def __init__(self, msg=None, data=None, filename=None, password=None, vals=None, file_obj=None): + self.n = None + self.e = None + self.d = None + self.p = None + self.q = None + if file_obj is not None: + self._from_private_key(file_obj, password) + return + if filename is not None: + self._from_private_key_file(filename, password) + return + if (msg is None) and (data is not None): + msg = Message(data) + if vals is not None: + self.e, self.n = vals + else: + if msg is None: + raise SSHException('Key object may not be empty') + if msg.get_string() != 'ssh-rsa': + raise SSHException('Invalid key') + self.e = msg.get_mpint() + self.n = msg.get_mpint() + self.size = util.bit_length(self.n) + + def __str__(self): + m = Message() + m.add_string('ssh-rsa') + m.add_mpint(self.e) + m.add_mpint(self.n) + return str(m) + + def __hash__(self): + h = hash(self.get_name()) + h = h * 37 + hash(self.e) + h = h * 37 + hash(self.n) + return hash(h) + + def get_name(self): + return 'ssh-rsa' + + def get_bits(self): + return self.size + + def can_sign(self): + return self.d is not None + + def sign_ssh_data(self, rpool, data): + digest = SHA.new(data).digest() + rsa = RSA.construct((long(self.n), long(self.e), long(self.d))) + sig = util.deflate_long(rsa.sign(self._pkcs1imify(digest), '')[0], 0) + m = Message() + m.add_string('ssh-rsa') + m.add_string(sig) + return m + + def verify_ssh_sig(self, data, msg): + if msg.get_string() != 'ssh-rsa': + return False + sig = util.inflate_long(msg.get_string(), True) + # verify the signature by SHA'ing the data and encrypting it using the + # public key. some wackiness ensues where we "pkcs1imify" the 20-byte + # hash into a string as long as the RSA key. + hash_obj = util.inflate_long(self._pkcs1imify(SHA.new(data).digest()), True) + rsa = RSA.construct((long(self.n), long(self.e))) + return rsa.verify(hash_obj, (sig,)) + + def _encode_key(self): + if (self.p is None) or (self.q is None): + raise SSHException('Not enough key info to write private key file') + keylist = [ 0, self.n, self.e, self.d, self.p, self.q, + self.d % (self.p - 1), self.d % (self.q - 1), + util.mod_inverse(self.q, self.p) ] + try: + b = BER() + b.encode(keylist) + except BERException: + raise SSHException('Unable to create ber encoding of key') + return str(b) + + def write_private_key_file(self, filename, password=None): + self._write_private_key_file('RSA', filename, self._encode_key(), password) + + def write_private_key(self, file_obj, password=None): + self._write_private_key('RSA', file_obj, self._encode_key(), password) + + def generate(bits, progress_func=None): + """ + Generate a new private RSA key. This factory function can be used to + generate a new host key or authentication key. + + @param bits: number of bits the generated key should be. + @type bits: int + @param progress_func: an optional function to call at key points in + key generation (used by C{pyCrypto.PublicKey}). + @type progress_func: function + @return: new private key + @rtype: L{RSAKey} + """ + rsa = RSA.generate(bits, rng.read, progress_func) + key = RSAKey(vals=(rsa.e, rsa.n)) + key.d = rsa.d + key.p = rsa.p + key.q = rsa.q + return key + generate = staticmethod(generate) + + + ### internals... + + + def _pkcs1imify(self, data): + """ + turn a 20-byte SHA1 hash into a blob of data as large as the key's N, + using PKCS1's \"emsa-pkcs1-v1_5\" encoding. totally bizarre. + """ + SHA1_DIGESTINFO = '\x30\x21\x30\x09\x06\x05\x2b\x0e\x03\x02\x1a\x05\x00\x04\x14' + size = len(util.deflate_long(self.n, 0)) + filler = '\xff' * (size - len(SHA1_DIGESTINFO) - len(data) - 3) + return '\x00\x01' + filler + '\x00' + SHA1_DIGESTINFO + data + + def _from_private_key_file(self, filename, password): + data = self._read_private_key_file('RSA', filename, password) + self._decode_key(data) + + def _from_private_key(self, file_obj, password): + data = self._read_private_key('RSA', file_obj, password) + self._decode_key(data) + + def _decode_key(self, data): + # private key file contains: + # RSAPrivateKey = { version = 0, n, e, d, p, q, d mod p-1, d mod q-1, q**-1 mod p } + try: + keylist = BER(data).decode() + except BERException: + raise SSHException('Unable to parse key file') + if (type(keylist) is not list) or (len(keylist) < 4) or (keylist[0] != 0): + raise SSHException('Not a valid RSA private key file (bad ber encoding)') + self.n = keylist[1] + self.e = keylist[2] + self.d = keylist[3] + # not really needed + self.p = keylist[4] + self.q = keylist[5] + self.size = util.bit_length(self.n) diff --git a/contrib/site-packages/paramiko/server.py b/contrib/site-packages/paramiko/server.py new file mode 100644 index 00000000..fdb40942 --- /dev/null +++ b/contrib/site-packages/paramiko/server.py @@ -0,0 +1,669 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +L{ServerInterface} is an interface to override for server support. +""" + +import threading +from paramiko.common import * +from paramiko import util + + +class InteractiveQuery (object): + """ + A query (set of prompts) for a user during interactive authentication. + """ + + def __init__(self, name='', instructions='', *prompts): + """ + Create a new interactive query to send to the client. The name and + instructions are optional, but are generally displayed to the end + user. A list of prompts may be included, or they may be added via + the L{add_prompt} method. + + @param name: name of this query + @type name: str + @param instructions: user instructions (usually short) about this query + @type instructions: str + @param prompts: one or more authentication prompts + @type prompts: str + """ + self.name = name + self.instructions = instructions + self.prompts = [] + for x in prompts: + if (type(x) is str) or (type(x) is unicode): + self.add_prompt(x) + else: + self.add_prompt(x[0], x[1]) + + def add_prompt(self, prompt, echo=True): + """ + Add a prompt to this query. The prompt should be a (reasonably short) + string. Multiple prompts can be added to the same query. + + @param prompt: the user prompt + @type prompt: str + @param echo: C{True} (default) if the user's response should be echoed; + C{False} if not (for a password or similar) + @type echo: bool + """ + self.prompts.append((prompt, echo)) + + +class ServerInterface (object): + """ + This class defines an interface for controlling the behavior of paramiko + in server mode. + + Methods on this class are called from paramiko's primary thread, so you + shouldn't do too much work in them. (Certainly nothing that blocks or + sleeps.) + """ + + def check_channel_request(self, kind, chanid): + """ + Determine if a channel request of a given type will be granted, and + return C{OPEN_SUCCEEDED} or an error code. This method is + called in server mode when the client requests a channel, after + authentication is complete. + + If you allow channel requests (and an ssh server that didn't would be + useless), you should also override some of the channel request methods + below, which are used to determine which services will be allowed on + a given channel: + - L{check_channel_pty_request} + - L{check_channel_shell_request} + - L{check_channel_subsystem_request} + - L{check_channel_window_change_request} + - L{check_channel_x11_request} + - L{check_channel_forward_agent_request} + + The C{chanid} parameter is a small number that uniquely identifies the + channel within a L{Transport}. A L{Channel} object is not created + unless this method returns C{OPEN_SUCCEEDED} -- once a + L{Channel} object is created, you can call L{Channel.get_id} to + retrieve the channel ID. + + The return value should either be C{OPEN_SUCCEEDED} (or + C{0}) to allow the channel request, or one of the following error + codes to reject it: + - C{OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED} + - C{OPEN_FAILED_CONNECT_FAILED} + - C{OPEN_FAILED_UNKNOWN_CHANNEL_TYPE} + - C{OPEN_FAILED_RESOURCE_SHORTAGE} + + The default implementation always returns + C{OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED}. + + @param kind: the kind of channel the client would like to open + (usually C{"session"}). + @type kind: str + @param chanid: ID of the channel + @type chanid: int + @return: a success or failure code (listed above) + @rtype: int + """ + return OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED + + def get_allowed_auths(self, username): + """ + Return a list of authentication methods supported by the server. + This list is sent to clients attempting to authenticate, to inform them + of authentication methods that might be successful. + + The "list" is actually a string of comma-separated names of types of + authentication. Possible values are C{"password"}, C{"publickey"}, + and C{"none"}. + + The default implementation always returns C{"password"}. + + @param username: the username requesting authentication. + @type username: str + @return: a comma-separated list of authentication types + @rtype: str + """ + return 'password' + + def check_auth_none(self, username): + """ + Determine if a client may open channels with no (further) + authentication. + + Return L{AUTH_FAILED} if the client must authenticate, or + L{AUTH_SUCCESSFUL} if it's okay for the client to not + authenticate. + + The default implementation always returns L{AUTH_FAILED}. + + @param username: the username of the client. + @type username: str + @return: L{AUTH_FAILED} if the authentication fails; + L{AUTH_SUCCESSFUL} if it succeeds. + @rtype: int + """ + return AUTH_FAILED + + def check_auth_password(self, username, password): + """ + Determine if a given username and password supplied by the client is + acceptable for use in authentication. + + Return L{AUTH_FAILED} if the password is not accepted, + L{AUTH_SUCCESSFUL} if the password is accepted and completes + the authentication, or L{AUTH_PARTIALLY_SUCCESSFUL} if your + authentication is stateful, and this key is accepted for + authentication, but more authentication is required. (In this latter + case, L{get_allowed_auths} will be called to report to the client what + options it has for continuing the authentication.) + + The default implementation always returns L{AUTH_FAILED}. + + @param username: the username of the authenticating client. + @type username: str + @param password: the password given by the client. + @type password: str + @return: L{AUTH_FAILED} if the authentication fails; + L{AUTH_SUCCESSFUL} if it succeeds; + L{AUTH_PARTIALLY_SUCCESSFUL} if the password auth is + successful, but authentication must continue. + @rtype: int + """ + return AUTH_FAILED + + def check_auth_publickey(self, username, key): + """ + Determine if a given key supplied by the client is acceptable for use + in authentication. You should override this method in server mode to + check the username and key and decide if you would accept a signature + made using this key. + + Return L{AUTH_FAILED} if the key is not accepted, + L{AUTH_SUCCESSFUL} if the key is accepted and completes the + authentication, or L{AUTH_PARTIALLY_SUCCESSFUL} if your + authentication is stateful, and this password is accepted for + authentication, but more authentication is required. (In this latter + case, L{get_allowed_auths} will be called to report to the client what + options it has for continuing the authentication.) + + Note that you don't have to actually verify any key signtature here. + If you're willing to accept the key, paramiko will do the work of + verifying the client's signature. + + The default implementation always returns L{AUTH_FAILED}. + + @param username: the username of the authenticating client + @type username: str + @param key: the key object provided by the client + @type key: L{PKey } + @return: L{AUTH_FAILED} if the client can't authenticate + with this key; L{AUTH_SUCCESSFUL} if it can; + L{AUTH_PARTIALLY_SUCCESSFUL} if it can authenticate with + this key but must continue with authentication + @rtype: int + """ + return AUTH_FAILED + + def check_auth_interactive(self, username, submethods): + """ + Begin an interactive authentication challenge, if supported. You + should override this method in server mode if you want to support the + C{"keyboard-interactive"} auth type, which requires you to send a + series of questions for the client to answer. + + Return L{AUTH_FAILED} if this auth method isn't supported. Otherwise, + you should return an L{InteractiveQuery} object containing the prompts + and instructions for the user. The response will be sent via a call + to L{check_auth_interactive_response}. + + The default implementation always returns L{AUTH_FAILED}. + + @param username: the username of the authenticating client + @type username: str + @param submethods: a comma-separated list of methods preferred by the + client (usually empty) + @type submethods: str + @return: L{AUTH_FAILED} if this auth method isn't supported; otherwise + an object containing queries for the user + @rtype: int or L{InteractiveQuery} + """ + return AUTH_FAILED + + def check_auth_interactive_response(self, responses): + """ + Continue or finish an interactive authentication challenge, if + supported. You should override this method in server mode if you want + to support the C{"keyboard-interactive"} auth type. + + Return L{AUTH_FAILED} if the responses are not accepted, + L{AUTH_SUCCESSFUL} if the responses are accepted and complete + the authentication, or L{AUTH_PARTIALLY_SUCCESSFUL} if your + authentication is stateful, and this set of responses is accepted for + authentication, but more authentication is required. (In this latter + case, L{get_allowed_auths} will be called to report to the client what + options it has for continuing the authentication.) + + If you wish to continue interactive authentication with more questions, + you may return an L{InteractiveQuery} object, which should cause the + client to respond with more answers, calling this method again. This + cycle can continue indefinitely. + + The default implementation always returns L{AUTH_FAILED}. + + @param responses: list of responses from the client + @type responses: list(str) + @return: L{AUTH_FAILED} if the authentication fails; + L{AUTH_SUCCESSFUL} if it succeeds; + L{AUTH_PARTIALLY_SUCCESSFUL} if the interactive auth is + successful, but authentication must continue; otherwise an object + containing queries for the user + @rtype: int or L{InteractiveQuery} + """ + return AUTH_FAILED + + def check_port_forward_request(self, address, port): + """ + Handle a request for port forwarding. The client is asking that + connections to the given address and port be forwarded back across + this ssh connection. An address of C{"0.0.0.0"} indicates a global + address (any address associated with this server) and a port of C{0} + indicates that no specific port is requested (usually the OS will pick + a port). + + The default implementation always returns C{False}, rejecting the + port forwarding request. If the request is accepted, you should return + the port opened for listening. + + @param address: the requested address + @type address: str + @param port: the requested port + @type port: int + @return: the port number that was opened for listening, or C{False} to + reject + @rtype: int + """ + return False + + def cancel_port_forward_request(self, address, port): + """ + The client would like to cancel a previous port-forwarding request. + If the given address and port is being forwarded across this ssh + connection, the port should be closed. + + @param address: the forwarded address + @type address: str + @param port: the forwarded port + @type port: int + """ + pass + + def check_global_request(self, kind, msg): + """ + Handle a global request of the given C{kind}. This method is called + in server mode and client mode, whenever the remote host makes a global + request. If there are any arguments to the request, they will be in + C{msg}. + + There aren't any useful global requests defined, aside from port + forwarding, so usually this type of request is an extension to the + protocol. + + If the request was successful and you would like to return contextual + data to the remote host, return a tuple. Items in the tuple will be + sent back with the successful result. (Note that the items in the + tuple can only be strings, ints, longs, or bools.) + + The default implementation always returns C{False}, indicating that it + does not support any global requests. + + @note: Port forwarding requests are handled separately, in + L{check_port_forward_request}. + + @param kind: the kind of global request being made. + @type kind: str + @param msg: any extra arguments to the request. + @type msg: L{Message} + @return: C{True} or a tuple of data if the request was granted; + C{False} otherwise. + @rtype: bool + """ + return False + + + ### Channel requests + + + def check_channel_pty_request(self, channel, term, width, height, pixelwidth, pixelheight, + modes): + """ + Determine if a pseudo-terminal of the given dimensions (usually + requested for shell access) can be provided on the given channel. + + The default implementation always returns C{False}. + + @param channel: the L{Channel} the pty request arrived on. + @type channel: L{Channel} + @param term: type of terminal requested (for example, C{"vt100"}). + @type term: str + @param width: width of screen in characters. + @type width: int + @param height: height of screen in characters. + @type height: int + @param pixelwidth: width of screen in pixels, if known (may be C{0} if + unknown). + @type pixelwidth: int + @param pixelheight: height of screen in pixels, if known (may be C{0} + if unknown). + @type pixelheight: int + @return: C{True} if the psuedo-terminal has been allocated; C{False} + otherwise. + @rtype: bool + """ + return False + + def check_channel_shell_request(self, channel): + """ + Determine if a shell will be provided to the client on the given + channel. If this method returns C{True}, the channel should be + connected to the stdin/stdout of a shell (or something that acts like + a shell). + + The default implementation always returns C{False}. + + @param channel: the L{Channel} the request arrived on. + @type channel: L{Channel} + @return: C{True} if this channel is now hooked up to a shell; C{False} + if a shell can't or won't be provided. + @rtype: bool + """ + return False + + def check_channel_exec_request(self, channel, command): + """ + Determine if a shell command will be executed for the client. If this + method returns C{True}, the channel should be connected to the stdin, + stdout, and stderr of the shell command. + + The default implementation always returns C{False}. + + @param channel: the L{Channel} the request arrived on. + @type channel: L{Channel} + @param command: the command to execute. + @type command: str + @return: C{True} if this channel is now hooked up to the stdin, + stdout, and stderr of the executing command; C{False} if the + command will not be executed. + @rtype: bool + + @since: 1.1 + """ + return False + + def check_channel_subsystem_request(self, channel, name): + """ + Determine if a requested subsystem will be provided to the client on + the given channel. If this method returns C{True}, all future I/O + through this channel will be assumed to be connected to the requested + subsystem. An example of a subsystem is C{sftp}. + + The default implementation checks for a subsystem handler assigned via + L{Transport.set_subsystem_handler}. + If one has been set, the handler is invoked and this method returns + C{True}. Otherwise it returns C{False}. + + @note: Because the default implementation uses the L{Transport} to + identify valid subsystems, you probably won't need to override this + method. + + @param channel: the L{Channel} the pty request arrived on. + @type channel: L{Channel} + @param name: name of the requested subsystem. + @type name: str + @return: C{True} if this channel is now hooked up to the requested + subsystem; C{False} if that subsystem can't or won't be provided. + @rtype: bool + """ + handler_class, larg, kwarg = channel.get_transport()._get_subsystem_handler(name) + if handler_class is None: + return False + handler = handler_class(channel, name, self, *larg, **kwarg) + handler.start() + return True + + def check_channel_window_change_request(self, channel, width, height, pixelwidth, pixelheight): + """ + Determine if the pseudo-terminal on the given channel can be resized. + This only makes sense if a pty was previously allocated on it. + + The default implementation always returns C{False}. + + @param channel: the L{Channel} the pty request arrived on. + @type channel: L{Channel} + @param width: width of screen in characters. + @type width: int + @param height: height of screen in characters. + @type height: int + @param pixelwidth: width of screen in pixels, if known (may be C{0} if + unknown). + @type pixelwidth: int + @param pixelheight: height of screen in pixels, if known (may be C{0} + if unknown). + @type pixelheight: int + @return: C{True} if the terminal was resized; C{False} if not. + @rtype: bool + """ + return False + + def check_channel_x11_request(self, channel, single_connection, auth_protocol, auth_cookie, screen_number): + """ + Determine if the client will be provided with an X11 session. If this + method returns C{True}, X11 applications should be routed through new + SSH channels, using L{Transport.open_x11_channel}. + + The default implementation always returns C{False}. + + @param channel: the L{Channel} the X11 request arrived on + @type channel: L{Channel} + @param single_connection: C{True} if only a single X11 channel should + be opened + @type single_connection: bool + @param auth_protocol: the protocol used for X11 authentication + @type auth_protocol: str + @param auth_cookie: the cookie used to authenticate to X11 + @type auth_cookie: str + @param screen_number: the number of the X11 screen to connect to + @type screen_number: int + @return: C{True} if the X11 session was opened; C{False} if not + @rtype: bool + """ + return False + + def check_channel_forward_agent_request(self, channel): + """ + Determine if the client will be provided with an forward agent session. + If this method returns C{True}, the server will allow SSH Agent + forwarding. + + The default implementation always returns C{False}. + + @param channel: the L{Channel} the request arrived on + @type channel: L{Channel} + @return: C{True} if the AgentForward was loaded; C{False} if not + @rtype: bool + """ + return False + + def check_channel_direct_tcpip_request(self, chanid, origin, destination): + """ + Determine if a local port forwarding channel will be granted, and + return C{OPEN_SUCCEEDED} or an error code. This method is + called in server mode when the client requests a channel, after + authentication is complete. + + The C{chanid} parameter is a small number that uniquely identifies the + channel within a L{Transport}. A L{Channel} object is not created + unless this method returns C{OPEN_SUCCEEDED} -- once a + L{Channel} object is created, you can call L{Channel.get_id} to + retrieve the channel ID. + + The origin and destination parameters are (ip_address, port) tuples + that correspond to both ends of the TCP connection in the forwarding + tunnel. + + The return value should either be C{OPEN_SUCCEEDED} (or + C{0}) to allow the channel request, or one of the following error + codes to reject it: + - C{OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED} + - C{OPEN_FAILED_CONNECT_FAILED} + - C{OPEN_FAILED_UNKNOWN_CHANNEL_TYPE} + - C{OPEN_FAILED_RESOURCE_SHORTAGE} + + The default implementation always returns + C{OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED}. + + @param chanid: ID of the channel + @type chanid: int + @param origin: 2-tuple containing the IP address and port of the + originator (client side) + @type origin: tuple + @param destination: 2-tuple containing the IP address and port of the + destination (server side) + @type destination: tuple + @return: a success or failure code (listed above) + @rtype: int + """ + return OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED + + def check_channel_env_request(self, channel, name, value): + """ + Check whether a given environment variable can be specified for the + given channel. This method should return C{True} if the server + is willing to set the specified environment variable. Note that + some environment variables (e.g., PATH) can be exceedingly + dangerous, so blindly allowing the client to set the environment + is almost certainly not a good idea. + + The default implementation always returns C{False}. + + @param channel: the L{Channel} the env request arrived on + @type channel: L{Channel} + @param name: foo bar baz + @type name: str + @param value: flklj + @type value: str + @rtype: bool + """ + return False + + +class SubsystemHandler (threading.Thread): + """ + Handler for a subsytem in server mode. If you create a subclass of this + class and pass it to + L{Transport.set_subsystem_handler}, + an object of this + class will be created for each request for this subsystem. Each new object + will be executed within its own new thread by calling L{start_subsystem}. + When that method completes, the channel is closed. + + For example, if you made a subclass C{MP3Handler} and registered it as the + handler for subsystem C{"mp3"}, then whenever a client has successfully + authenticated and requests subsytem C{"mp3"}, an object of class + C{MP3Handler} will be created, and L{start_subsystem} will be called on + it from a new thread. + """ + def __init__(self, channel, name, server): + """ + Create a new handler for a channel. This is used by L{ServerInterface} + to start up a new handler when a channel requests this subsystem. You + don't need to override this method, but if you do, be sure to pass the + C{channel} and C{name} parameters through to the original C{__init__} + method here. + + @param channel: the channel associated with this subsystem request. + @type channel: L{Channel} + @param name: name of the requested subsystem. + @type name: str + @param server: the server object for the session that started this + subsystem + @type server: L{ServerInterface} + """ + threading.Thread.__init__(self, target=self._run) + self.__channel = channel + self.__transport = channel.get_transport() + self.__name = name + self.__server = server + + def get_server(self): + """ + Return the L{ServerInterface} object associated with this channel and + subsystem. + + @rtype: L{ServerInterface} + """ + return self.__server + + def _run(self): + try: + self.__transport._log(DEBUG, 'Starting handler for subsystem %s' % self.__name) + self.start_subsystem(self.__name, self.__transport, self.__channel) + except Exception, e: + self.__transport._log(ERROR, 'Exception in subsystem handler for "%s": %s' % + (self.__name, str(e))) + self.__transport._log(ERROR, util.tb_strings()) + try: + self.finish_subsystem() + except: + pass + + def start_subsystem(self, name, transport, channel): + """ + Process an ssh subsystem in server mode. This method is called on a + new object (and in a new thread) for each subsystem request. It is + assumed that all subsystem logic will take place here, and when the + subsystem is finished, this method will return. After this method + returns, the channel is closed. + + The combination of C{transport} and C{channel} are unique; this handler + corresponds to exactly one L{Channel} on one L{Transport}. + + @note: It is the responsibility of this method to exit if the + underlying L{Transport} is closed. This can be done by checking + L{Transport.is_active} or noticing an EOF + on the L{Channel}. If this method loops forever without checking + for this case, your python interpreter may refuse to exit because + this thread will still be running. + + @param name: name of the requested subsystem. + @type name: str + @param transport: the server-mode L{Transport}. + @type transport: L{Transport} + @param channel: the channel associated with this subsystem request. + @type channel: L{Channel} + """ + pass + + def finish_subsystem(self): + """ + Perform any cleanup at the end of a subsystem. The default + implementation just closes the channel. + + @since: 1.1 + """ + self.__channel.close() diff --git a/contrib/site-packages/paramiko/sftp.py b/contrib/site-packages/paramiko/sftp.py new file mode 100644 index 00000000..a97c300f --- /dev/null +++ b/contrib/site-packages/paramiko/sftp.py @@ -0,0 +1,188 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +import select +import socket +import struct + +from paramiko.common import * +from paramiko import util +from paramiko.channel import Channel +from paramiko.message import Message + + +CMD_INIT, CMD_VERSION, CMD_OPEN, CMD_CLOSE, CMD_READ, CMD_WRITE, CMD_LSTAT, CMD_FSTAT, \ + CMD_SETSTAT, CMD_FSETSTAT, CMD_OPENDIR, CMD_READDIR, CMD_REMOVE, CMD_MKDIR, \ + CMD_RMDIR, CMD_REALPATH, CMD_STAT, CMD_RENAME, CMD_READLINK, CMD_SYMLINK \ + = range(1, 21) +CMD_STATUS, CMD_HANDLE, CMD_DATA, CMD_NAME, CMD_ATTRS = range(101, 106) +CMD_EXTENDED, CMD_EXTENDED_REPLY = range(200, 202) + +SFTP_OK = 0 +SFTP_EOF, SFTP_NO_SUCH_FILE, SFTP_PERMISSION_DENIED, SFTP_FAILURE, SFTP_BAD_MESSAGE, \ + SFTP_NO_CONNECTION, SFTP_CONNECTION_LOST, SFTP_OP_UNSUPPORTED = range(1, 9) + +SFTP_DESC = [ 'Success', + 'End of file', + 'No such file', + 'Permission denied', + 'Failure', + 'Bad message', + 'No connection', + 'Connection lost', + 'Operation unsupported' ] + +SFTP_FLAG_READ = 0x1 +SFTP_FLAG_WRITE = 0x2 +SFTP_FLAG_APPEND = 0x4 +SFTP_FLAG_CREATE = 0x8 +SFTP_FLAG_TRUNC = 0x10 +SFTP_FLAG_EXCL = 0x20 + +_VERSION = 3 + + +# for debugging +CMD_NAMES = { + CMD_INIT: 'init', + CMD_VERSION: 'version', + CMD_OPEN: 'open', + CMD_CLOSE: 'close', + CMD_READ: 'read', + CMD_WRITE: 'write', + CMD_LSTAT: 'lstat', + CMD_FSTAT: 'fstat', + CMD_SETSTAT: 'setstat', + CMD_FSETSTAT: 'fsetstat', + CMD_OPENDIR: 'opendir', + CMD_READDIR: 'readdir', + CMD_REMOVE: 'remove', + CMD_MKDIR: 'mkdir', + CMD_RMDIR: 'rmdir', + CMD_REALPATH: 'realpath', + CMD_STAT: 'stat', + CMD_RENAME: 'rename', + CMD_READLINK: 'readlink', + CMD_SYMLINK: 'symlink', + CMD_STATUS: 'status', + CMD_HANDLE: 'handle', + CMD_DATA: 'data', + CMD_NAME: 'name', + CMD_ATTRS: 'attrs', + CMD_EXTENDED: 'extended', + CMD_EXTENDED_REPLY: 'extended_reply' + } + + +class SFTPError (Exception): + pass + + +class BaseSFTP (object): + def __init__(self): + self.logger = util.get_logger('paramiko.sftp') + self.sock = None + self.ultra_debug = False + + + ### internals... + + + def _send_version(self): + self._send_packet(CMD_INIT, struct.pack('>I', _VERSION)) + t, data = self._read_packet() + if t != CMD_VERSION: + raise SFTPError('Incompatible sftp protocol') + version = struct.unpack('>I', data[:4])[0] + # if version != _VERSION: + # raise SFTPError('Incompatible sftp protocol') + return version + + def _send_server_version(self): + # winscp will freak out if the server sends version info before the + # client finishes sending INIT. + t, data = self._read_packet() + if t != CMD_INIT: + raise SFTPError('Incompatible sftp protocol') + version = struct.unpack('>I', data[:4])[0] + # advertise that we support "check-file" + extension_pairs = [ 'check-file', 'md5,sha1' ] + msg = Message() + msg.add_int(_VERSION) + msg.add(*extension_pairs) + self._send_packet(CMD_VERSION, str(msg)) + return version + + def _log(self, level, msg, *args): + self.logger.log(level, msg, *args) + + def _write_all(self, out): + while len(out) > 0: + n = self.sock.send(out) + if n <= 0: + raise EOFError() + if n == len(out): + return + out = out[n:] + return + + def _read_all(self, n): + out = '' + while n > 0: + if isinstance(self.sock, socket.socket): + # sometimes sftp is used directly over a socket instead of + # through a paramiko channel. in this case, check periodically + # if the socket is closed. (for some reason, recv() won't ever + # return or raise an exception, but calling select on a closed + # socket will.) + while True: + read, write, err = select.select([ self.sock ], [], [], 0.1) + if len(read) > 0: + x = self.sock.recv(n) + break + else: + x = self.sock.recv(n) + + if len(x) == 0: + raise EOFError() + out += x + n -= len(x) + return out + + def _send_packet(self, t, packet): + #self._log(DEBUG2, 'write: %s (len=%d)' % (CMD_NAMES.get(t, '0x%02x' % t), len(packet))) + out = struct.pack('>I', len(packet) + 1) + chr(t) + packet + if self.ultra_debug: + self._log(DEBUG, util.format_binary(out, 'OUT: ')) + self._write_all(out) + + def _read_packet(self): + x = self._read_all(4) + # most sftp servers won't accept packets larger than about 32k, so + # anything with the high byte set (> 16MB) is just garbage. + if x[0] != '\x00': + raise SFTPError('Garbage packet received') + size = struct.unpack('>I', x)[0] + data = self._read_all(size) + if self.ultra_debug: + self._log(DEBUG, util.format_binary(data, 'IN: ')); + if size > 0: + t = ord(data[0]) + #self._log(DEBUG2, 'read: %s (len=%d)' % (CMD_NAMES.get(t), '0x%02x' % t, len(data)-1)) + return t, data[1:] + return 0, '' diff --git a/contrib/site-packages/paramiko/sftp_attr.py b/contrib/site-packages/paramiko/sftp_attr.py new file mode 100644 index 00000000..b459b04b --- /dev/null +++ b/contrib/site-packages/paramiko/sftp_attr.py @@ -0,0 +1,223 @@ +# Copyright (C) 2003-2006 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +import stat +import time +from paramiko.common import * +from paramiko.sftp import * + + +class SFTPAttributes (object): + """ + Representation of the attributes of a file (or proxied file) for SFTP in + client or server mode. It attemps to mirror the object returned by + C{os.stat} as closely as possible, so it may have the following fields, + with the same meanings as those returned by an C{os.stat} object: + - st_size + - st_uid + - st_gid + - st_mode + - st_atime + - st_mtime + + Because SFTP allows flags to have other arbitrary named attributes, these + are stored in a dict named C{attr}. Occasionally, the filename is also + stored, in C{filename}. + """ + + FLAG_SIZE = 1 + FLAG_UIDGID = 2 + FLAG_PERMISSIONS = 4 + FLAG_AMTIME = 8 + FLAG_EXTENDED = 0x80000000L + + def __init__(self): + """ + Create a new (empty) SFTPAttributes object. All fields will be empty. + """ + self._flags = 0 + self.st_size = None + self.st_uid = None + self.st_gid = None + self.st_mode = None + self.st_atime = None + self.st_mtime = None + self.attr = {} + + def from_stat(cls, obj, filename=None): + """ + Create an SFTPAttributes object from an existing C{stat} object (an + object returned by C{os.stat}). + + @param obj: an object returned by C{os.stat} (or equivalent). + @type obj: object + @param filename: the filename associated with this file. + @type filename: str + @return: new L{SFTPAttributes} object with the same attribute fields. + @rtype: L{SFTPAttributes} + """ + attr = cls() + attr.st_size = obj.st_size + attr.st_uid = obj.st_uid + attr.st_gid = obj.st_gid + attr.st_mode = obj.st_mode + attr.st_atime = obj.st_atime + attr.st_mtime = obj.st_mtime + if filename is not None: + attr.filename = filename + return attr + from_stat = classmethod(from_stat) + + def __repr__(self): + return '' % self._debug_str() + + + ### internals... + + + def _from_msg(cls, msg, filename=None, longname=None): + attr = cls() + attr._unpack(msg) + if filename is not None: + attr.filename = filename + if longname is not None: + attr.longname = longname + return attr + _from_msg = classmethod(_from_msg) + + def _unpack(self, msg): + self._flags = msg.get_int() + if self._flags & self.FLAG_SIZE: + self.st_size = msg.get_int64() + if self._flags & self.FLAG_UIDGID: + self.st_uid = msg.get_int() + self.st_gid = msg.get_int() + if self._flags & self.FLAG_PERMISSIONS: + self.st_mode = msg.get_int() + if self._flags & self.FLAG_AMTIME: + self.st_atime = msg.get_int() + self.st_mtime = msg.get_int() + if self._flags & self.FLAG_EXTENDED: + count = msg.get_int() + for i in range(count): + self.attr[msg.get_string()] = msg.get_string() + + def _pack(self, msg): + self._flags = 0 + if self.st_size is not None: + self._flags |= self.FLAG_SIZE + if (self.st_uid is not None) and (self.st_gid is not None): + self._flags |= self.FLAG_UIDGID + if self.st_mode is not None: + self._flags |= self.FLAG_PERMISSIONS + if (self.st_atime is not None) and (self.st_mtime is not None): + self._flags |= self.FLAG_AMTIME + if len(self.attr) > 0: + self._flags |= self.FLAG_EXTENDED + msg.add_int(self._flags) + if self._flags & self.FLAG_SIZE: + msg.add_int64(self.st_size) + if self._flags & self.FLAG_UIDGID: + msg.add_int(self.st_uid) + msg.add_int(self.st_gid) + if self._flags & self.FLAG_PERMISSIONS: + msg.add_int(self.st_mode) + if self._flags & self.FLAG_AMTIME: + # throw away any fractional seconds + msg.add_int(long(self.st_atime)) + msg.add_int(long(self.st_mtime)) + if self._flags & self.FLAG_EXTENDED: + msg.add_int(len(self.attr)) + for key, val in self.attr.iteritems(): + msg.add_string(key) + msg.add_string(val) + return + + def _debug_str(self): + out = '[ ' + if self.st_size is not None: + out += 'size=%d ' % self.st_size + if (self.st_uid is not None) and (self.st_gid is not None): + out += 'uid=%d gid=%d ' % (self.st_uid, self.st_gid) + if self.st_mode is not None: + out += 'mode=' + oct(self.st_mode) + ' ' + if (self.st_atime is not None) and (self.st_mtime is not None): + out += 'atime=%d mtime=%d ' % (self.st_atime, self.st_mtime) + for k, v in self.attr.iteritems(): + out += '"%s"=%r ' % (str(k), v) + out += ']' + return out + + def _rwx(n, suid, sticky=False): + if suid: + suid = 2 + out = '-r'[n >> 2] + '-w'[(n >> 1) & 1] + if sticky: + out += '-xTt'[suid + (n & 1)] + else: + out += '-xSs'[suid + (n & 1)] + return out + _rwx = staticmethod(_rwx) + + def __str__(self): + "create a unix-style long description of the file (like ls -l)" + if self.st_mode is not None: + kind = stat.S_IFMT(self.st_mode) + if kind == stat.S_IFIFO: + ks = 'p' + elif kind == stat.S_IFCHR: + ks = 'c' + elif kind == stat.S_IFDIR: + ks = 'd' + elif kind == stat.S_IFBLK: + ks = 'b' + elif kind == stat.S_IFREG: + ks = '-' + elif kind == stat.S_IFLNK: + ks = 'l' + elif kind == stat.S_IFSOCK: + ks = 's' + else: + ks = '?' + ks += self._rwx((self.st_mode & 0700) >> 6, self.st_mode & stat.S_ISUID) + ks += self._rwx((self.st_mode & 070) >> 3, self.st_mode & stat.S_ISGID) + ks += self._rwx(self.st_mode & 7, self.st_mode & stat.S_ISVTX, True) + else: + ks = '?---------' + # compute display date + if (self.st_mtime is None) or (self.st_mtime == 0xffffffffL): + # shouldn't really happen + datestr = '(unknown date)' + else: + if abs(time.time() - self.st_mtime) > 15552000: + # (15552000 = 6 months) + datestr = time.strftime('%d %b %Y', time.localtime(self.st_mtime)) + else: + datestr = time.strftime('%d %b %H:%M', time.localtime(self.st_mtime)) + filename = getattr(self, 'filename', '?') + + # not all servers support uid/gid + uid = self.st_uid + gid = self.st_gid + if uid is None: + uid = 0 + if gid is None: + gid = 0 + + return '%s 1 %-8d %-8d %8d %-12s %s' % (ks, uid, gid, self.st_size, datestr, filename) + diff --git a/contrib/site-packages/paramiko/sftp_client.py b/contrib/site-packages/paramiko/sftp_client.py new file mode 100644 index 00000000..d9215743 --- /dev/null +++ b/contrib/site-packages/paramiko/sftp_client.py @@ -0,0 +1,787 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +Client-mode SFTP support. +""" + +from binascii import hexlify +import errno +import os +import stat +import threading +import time +import weakref + +from paramiko.sftp import * +from paramiko.sftp_attr import SFTPAttributes +from paramiko.ssh_exception import SSHException +from paramiko.sftp_file import SFTPFile + + +def _to_unicode(s): + """ + decode a string as ascii or utf8 if possible (as required by the sftp + protocol). if neither works, just return a byte string because the server + probably doesn't know the filename's encoding. + """ + try: + return s.encode('ascii') + except UnicodeError: + try: + return s.decode('utf-8') + except UnicodeError: + return s + + +class SFTPClient (BaseSFTP): + """ + SFTP client object. C{SFTPClient} is used to open an sftp session across + an open ssh L{Transport} and do remote file operations. + """ + + def __init__(self, sock): + """ + Create an SFTP client from an existing L{Channel}. The channel + should already have requested the C{"sftp"} subsystem. + + An alternate way to create an SFTP client context is by using + L{from_transport}. + + @param sock: an open L{Channel} using the C{"sftp"} subsystem + @type sock: L{Channel} + + @raise SSHException: if there's an exception while negotiating + sftp + """ + BaseSFTP.__init__(self) + self.sock = sock + self.ultra_debug = False + self.request_number = 1 + # lock for request_number + self._lock = threading.Lock() + self._cwd = None + # request # -> SFTPFile + self._expecting = weakref.WeakValueDictionary() + if type(sock) is Channel: + # override default logger + transport = self.sock.get_transport() + self.logger = util.get_logger(transport.get_log_channel() + '.sftp') + self.ultra_debug = transport.get_hexdump() + try: + server_version = self._send_version() + except EOFError, x: + raise SSHException('EOF during negotiation') + self._log(INFO, 'Opened sftp connection (server version %d)' % server_version) + + def from_transport(cls, t): + """ + Create an SFTP client channel from an open L{Transport}. + + @param t: an open L{Transport} which is already authenticated + @type t: L{Transport} + @return: a new L{SFTPClient} object, referring to an sftp session + (channel) across the transport + @rtype: L{SFTPClient} + """ + chan = t.open_session() + if chan is None: + return None + chan.invoke_subsystem('sftp') + return cls(chan) + from_transport = classmethod(from_transport) + + def _log(self, level, msg, *args): + if isinstance(msg, list): + for m in msg: + super(SFTPClient, self)._log(level, "[chan %s] " + m, *([ self.sock.get_name() ] + list(args))) + else: + super(SFTPClient, self)._log(level, "[chan %s] " + msg, *([ self.sock.get_name() ] + list(args))) + + def close(self): + """ + Close the SFTP session and its underlying channel. + + @since: 1.4 + """ + self._log(INFO, 'sftp session closed.') + self.sock.close() + + def get_channel(self): + """ + Return the underlying L{Channel} object for this SFTP session. This + might be useful for doing things like setting a timeout on the channel. + + @return: the SSH channel + @rtype: L{Channel} + + @since: 1.7.1 + """ + return self.sock + + def listdir(self, path='.'): + """ + Return a list containing the names of the entries in the given C{path}. + The list is in arbitrary order. It does not include the special + entries C{'.'} and C{'..'} even if they are present in the folder. + This method is meant to mirror C{os.listdir} as closely as possible. + For a list of full L{SFTPAttributes} objects, see L{listdir_attr}. + + @param path: path to list (defaults to C{'.'}) + @type path: str + @return: list of filenames + @rtype: list of str + """ + return [f.filename for f in self.listdir_attr(path)] + + def listdir_attr(self, path='.'): + """ + Return a list containing L{SFTPAttributes} objects corresponding to + files in the given C{path}. The list is in arbitrary order. It does + not include the special entries C{'.'} and C{'..'} even if they are + present in the folder. + + The returned L{SFTPAttributes} objects will each have an additional + field: C{longname}, which may contain a formatted string of the file's + attributes, in unix format. The content of this string will probably + depend on the SFTP server implementation. + + @param path: path to list (defaults to C{'.'}) + @type path: str + @return: list of attributes + @rtype: list of L{SFTPAttributes} + + @since: 1.2 + """ + path = self._adjust_cwd(path) + self._log(DEBUG, 'listdir(%r)' % path) + t, msg = self._request(CMD_OPENDIR, path) + if t != CMD_HANDLE: + raise SFTPError('Expected handle') + handle = msg.get_string() + filelist = [] + while True: + try: + t, msg = self._request(CMD_READDIR, handle) + except EOFError, e: + # done with handle + break + if t != CMD_NAME: + raise SFTPError('Expected name response') + count = msg.get_int() + for i in range(count): + filename = _to_unicode(msg.get_string()) + longname = _to_unicode(msg.get_string()) + attr = SFTPAttributes._from_msg(msg, filename, longname) + if (filename != '.') and (filename != '..'): + filelist.append(attr) + self._request(CMD_CLOSE, handle) + return filelist + + def open(self, filename, mode='r', bufsize=-1): + """ + Open a file on the remote server. The arguments are the same as for + python's built-in C{file} (aka C{open}). A file-like object is + returned, which closely mimics the behavior of a normal python file + object, including the ability to be used as a context manager. + + The mode indicates how the file is to be opened: C{'r'} for reading, + C{'w'} for writing (truncating an existing file), C{'a'} for appending, + C{'r+'} for reading/writing, C{'w+'} for reading/writing (truncating an + existing file), C{'a+'} for reading/appending. The python C{'b'} flag + is ignored, since SSH treats all files as binary. The C{'U'} flag is + supported in a compatible way. + + Since 1.5.2, an C{'x'} flag indicates that the operation should only + succeed if the file was created and did not previously exist. This has + no direct mapping to python's file flags, but is commonly known as the + C{O_EXCL} flag in posix. + + The file will be buffered in standard python style by default, but + can be altered with the C{bufsize} parameter. C{0} turns off + buffering, C{1} uses line buffering, and any number greater than 1 + (C{>1}) uses that specific buffer size. + + @param filename: name of the file to open + @type filename: str + @param mode: mode (python-style) to open in + @type mode: str + @param bufsize: desired buffering (-1 = default buffer size) + @type bufsize: int + @return: a file object representing the open file + @rtype: SFTPFile + + @raise IOError: if the file could not be opened. + """ + filename = self._adjust_cwd(filename) + self._log(DEBUG, 'open(%r, %r)' % (filename, mode)) + imode = 0 + if ('r' in mode) or ('+' in mode): + imode |= SFTP_FLAG_READ + if ('w' in mode) or ('+' in mode) or ('a' in mode): + imode |= SFTP_FLAG_WRITE + if ('w' in mode): + imode |= SFTP_FLAG_CREATE | SFTP_FLAG_TRUNC + if ('a' in mode): + imode |= SFTP_FLAG_CREATE | SFTP_FLAG_APPEND + if ('x' in mode): + imode |= SFTP_FLAG_CREATE | SFTP_FLAG_EXCL + attrblock = SFTPAttributes() + t, msg = self._request(CMD_OPEN, filename, imode, attrblock) + if t != CMD_HANDLE: + raise SFTPError('Expected handle') + handle = msg.get_string() + self._log(DEBUG, 'open(%r, %r) -> %s' % (filename, mode, hexlify(handle))) + return SFTPFile(self, handle, mode, bufsize) + + # python continues to vacillate about "open" vs "file"... + file = open + + def remove(self, path): + """ + Remove the file at the given path. This only works on files; for + removing folders (directories), use L{rmdir}. + + @param path: path (absolute or relative) of the file to remove + @type path: str + + @raise IOError: if the path refers to a folder (directory) + """ + path = self._adjust_cwd(path) + self._log(DEBUG, 'remove(%r)' % path) + self._request(CMD_REMOVE, path) + + unlink = remove + + def rename(self, oldpath, newpath): + """ + Rename a file or folder from C{oldpath} to C{newpath}. + + @param oldpath: existing name of the file or folder + @type oldpath: str + @param newpath: new name for the file or folder + @type newpath: str + + @raise IOError: if C{newpath} is a folder, or something else goes + wrong + """ + oldpath = self._adjust_cwd(oldpath) + newpath = self._adjust_cwd(newpath) + self._log(DEBUG, 'rename(%r, %r)' % (oldpath, newpath)) + self._request(CMD_RENAME, oldpath, newpath) + + def mkdir(self, path, mode=0777): + """ + Create a folder (directory) named C{path} with numeric mode C{mode}. + The default mode is 0777 (octal). On some systems, mode is ignored. + Where it is used, the current umask value is first masked out. + + @param path: name of the folder to create + @type path: str + @param mode: permissions (posix-style) for the newly-created folder + @type mode: int + """ + path = self._adjust_cwd(path) + self._log(DEBUG, 'mkdir(%r, %r)' % (path, mode)) + attr = SFTPAttributes() + attr.st_mode = mode + self._request(CMD_MKDIR, path, attr) + + def rmdir(self, path): + """ + Remove the folder named C{path}. + + @param path: name of the folder to remove + @type path: str + """ + path = self._adjust_cwd(path) + self._log(DEBUG, 'rmdir(%r)' % path) + self._request(CMD_RMDIR, path) + + def stat(self, path): + """ + Retrieve information about a file on the remote system. The return + value is an object whose attributes correspond to the attributes of + python's C{stat} structure as returned by C{os.stat}, except that it + contains fewer fields. An SFTP server may return as much or as little + info as it wants, so the results may vary from server to server. + + Unlike a python C{stat} object, the result may not be accessed as a + tuple. This is mostly due to the author's slack factor. + + The fields supported are: C{st_mode}, C{st_size}, C{st_uid}, C{st_gid}, + C{st_atime}, and C{st_mtime}. + + @param path: the filename to stat + @type path: str + @return: an object containing attributes about the given file + @rtype: SFTPAttributes + """ + path = self._adjust_cwd(path) + self._log(DEBUG, 'stat(%r)' % path) + t, msg = self._request(CMD_STAT, path) + if t != CMD_ATTRS: + raise SFTPError('Expected attributes') + return SFTPAttributes._from_msg(msg) + + def lstat(self, path): + """ + Retrieve information about a file on the remote system, without + following symbolic links (shortcuts). This otherwise behaves exactly + the same as L{stat}. + + @param path: the filename to stat + @type path: str + @return: an object containing attributes about the given file + @rtype: SFTPAttributes + """ + path = self._adjust_cwd(path) + self._log(DEBUG, 'lstat(%r)' % path) + t, msg = self._request(CMD_LSTAT, path) + if t != CMD_ATTRS: + raise SFTPError('Expected attributes') + return SFTPAttributes._from_msg(msg) + + def symlink(self, source, dest): + """ + Create a symbolic link (shortcut) of the C{source} path at + C{destination}. + + @param source: path of the original file + @type source: str + @param dest: path of the newly created symlink + @type dest: str + """ + dest = self._adjust_cwd(dest) + self._log(DEBUG, 'symlink(%r, %r)' % (source, dest)) + if type(source) is unicode: + source = source.encode('utf-8') + self._request(CMD_SYMLINK, source, dest) + + def chmod(self, path, mode): + """ + Change the mode (permissions) of a file. The permissions are + unix-style and identical to those used by python's C{os.chmod} + function. + + @param path: path of the file to change the permissions of + @type path: str + @param mode: new permissions + @type mode: int + """ + path = self._adjust_cwd(path) + self._log(DEBUG, 'chmod(%r, %r)' % (path, mode)) + attr = SFTPAttributes() + attr.st_mode = mode + self._request(CMD_SETSTAT, path, attr) + + def chown(self, path, uid, gid): + """ + Change the owner (C{uid}) and group (C{gid}) of a file. As with + python's C{os.chown} function, you must pass both arguments, so if you + only want to change one, use L{stat} first to retrieve the current + owner and group. + + @param path: path of the file to change the owner and group of + @type path: str + @param uid: new owner's uid + @type uid: int + @param gid: new group id + @type gid: int + """ + path = self._adjust_cwd(path) + self._log(DEBUG, 'chown(%r, %r, %r)' % (path, uid, gid)) + attr = SFTPAttributes() + attr.st_uid, attr.st_gid = uid, gid + self._request(CMD_SETSTAT, path, attr) + + def utime(self, path, times): + """ + Set the access and modified times of the file specified by C{path}. If + C{times} is C{None}, then the file's access and modified times are set + to the current time. Otherwise, C{times} must be a 2-tuple of numbers, + of the form C{(atime, mtime)}, which is used to set the access and + modified times, respectively. This bizarre API is mimicked from python + for the sake of consistency -- I apologize. + + @param path: path of the file to modify + @type path: str + @param times: C{None} or a tuple of (access time, modified time) in + standard internet epoch time (seconds since 01 January 1970 GMT) + @type times: tuple(int) + """ + path = self._adjust_cwd(path) + if times is None: + times = (time.time(), time.time()) + self._log(DEBUG, 'utime(%r, %r)' % (path, times)) + attr = SFTPAttributes() + attr.st_atime, attr.st_mtime = times + self._request(CMD_SETSTAT, path, attr) + + def truncate(self, path, size): + """ + Change the size of the file specified by C{path}. This usually extends + or shrinks the size of the file, just like the C{truncate()} method on + python file objects. + + @param path: path of the file to modify + @type path: str + @param size: the new size of the file + @type size: int or long + """ + path = self._adjust_cwd(path) + self._log(DEBUG, 'truncate(%r, %r)' % (path, size)) + attr = SFTPAttributes() + attr.st_size = size + self._request(CMD_SETSTAT, path, attr) + + def readlink(self, path): + """ + Return the target of a symbolic link (shortcut). You can use + L{symlink} to create these. The result may be either an absolute or + relative pathname. + + @param path: path of the symbolic link file + @type path: str + @return: target path + @rtype: str + """ + path = self._adjust_cwd(path) + self._log(DEBUG, 'readlink(%r)' % path) + t, msg = self._request(CMD_READLINK, path) + if t != CMD_NAME: + raise SFTPError('Expected name response') + count = msg.get_int() + if count == 0: + return None + if count != 1: + raise SFTPError('Readlink returned %d results' % count) + return _to_unicode(msg.get_string()) + + def normalize(self, path): + """ + Return the normalized path (on the server) of a given path. This + can be used to quickly resolve symbolic links or determine what the + server is considering to be the "current folder" (by passing C{'.'} + as C{path}). + + @param path: path to be normalized + @type path: str + @return: normalized form of the given path + @rtype: str + + @raise IOError: if the path can't be resolved on the server + """ + path = self._adjust_cwd(path) + self._log(DEBUG, 'normalize(%r)' % path) + t, msg = self._request(CMD_REALPATH, path) + if t != CMD_NAME: + raise SFTPError('Expected name response') + count = msg.get_int() + if count != 1: + raise SFTPError('Realpath returned %d results' % count) + return _to_unicode(msg.get_string()) + + def chdir(self, path): + """ + Change the "current directory" of this SFTP session. Since SFTP + doesn't really have the concept of a current working directory, this + is emulated by paramiko. Once you use this method to set a working + directory, all operations on this SFTPClient object will be relative + to that path. You can pass in C{None} to stop using a current working + directory. + + @param path: new current working directory + @type path: str + + @raise IOError: if the requested path doesn't exist on the server + + @since: 1.4 + """ + if path is None: + self._cwd = None + return + if not stat.S_ISDIR(self.stat(path).st_mode): + raise SFTPError(errno.ENOTDIR, "%s: %s" % (os.strerror(errno.ENOTDIR), path)) + self._cwd = self.normalize(path).encode('utf-8') + + def getcwd(self): + """ + Return the "current working directory" for this SFTP session, as + emulated by paramiko. If no directory has been set with L{chdir}, + this method will return C{None}. + + @return: the current working directory on the server, or C{None} + @rtype: str + + @since: 1.4 + """ + return self._cwd + + def putfo(self, fl, remotepath, file_size=0, callback=None, confirm=True): + """ + Copy the contents of an open file object (C{fl}) to the SFTP server as + C{remotepath}. Any exception raised by operations will be passed through. + + The SFTP operations use pipelining for speed. + + @param fl: opened file or file-like object to copy + @type localpath: object + @param remotepath: the destination path on the SFTP server + @type remotepath: str + @param file_size: optional size parameter passed to callback. If none is + specified, size defaults to 0 + @type file_size: int + @param callback: optional callback function that accepts the bytes + transferred so far and the total bytes to be transferred + (since 1.7.4) + @type callback: function(int, int) + @param confirm: whether to do a stat() on the file afterwards to + confirm the file size (since 1.7.7) + @type confirm: bool + + @return: an object containing attributes about the given file + (since 1.7.4) + @rtype: SFTPAttributes + + @since: 1.4 + """ + fr = self.file(remotepath, 'wb') + fr.set_pipelined(True) + size = 0 + try: + while True: + data = fl.read(32768) + fr.write(data) + size += len(data) + if callback is not None: + callback(size, file_size) + if len(data) == 0: + break + finally: + fr.close() + if confirm: + s = self.stat(remotepath) + if s.st_size != size: + raise IOError('size mismatch in put! %d != %d' % (s.st_size, size)) + else: + s = SFTPAttributes() + return s + + def put(self, localpath, remotepath, callback=None, confirm=True): + """ + Copy a local file (C{localpath}) to the SFTP server as C{remotepath}. + Any exception raised by operations will be passed through. This + method is primarily provided as a convenience. + + The SFTP operations use pipelining for speed. + + @param localpath: the local file to copy + @type localpath: str + @param remotepath: the destination path on the SFTP server + @type remotepath: str + @param callback: optional callback function that accepts the bytes + transferred so far and the total bytes to be transferred + (since 1.7.4) + @type callback: function(int, int) + @param confirm: whether to do a stat() on the file afterwards to + confirm the file size (since 1.7.7) + @type confirm: bool + + @return: an object containing attributes about the given file + (since 1.7.4) + @rtype: SFTPAttributes + + @since: 1.4 + """ + file_size = os.stat(localpath).st_size + fl = file(localpath, 'rb') + try: + return self.putfo(fl, remotepath, os.stat(localpath).st_size, callback, confirm) + finally: + fl.close() + + def getfo(self, remotepath, fl, callback=None): + """ + Copy a remote file (C{remotepath}) from the SFTP server and write to + an open file or file-like object, C{fl}. Any exception raised by + operations will be passed through. This method is primarily provided + as a convenience. + + @param remotepath: opened file or file-like object to copy to + @type remotepath: object + @param fl: the destination path on the local host or open file + object + @type localpath: str + @param callback: optional callback function that accepts the bytes + transferred so far and the total bytes to be transferred + (since 1.7.4) + @type callback: function(int, int) + @return: the number of bytes written to the opened file object + + @since: 1.4 + """ + fr = self.file(remotepath, 'rb') + file_size = self.stat(remotepath).st_size + fr.prefetch() + try: + size = 0 + while True: + data = fr.read(32768) + fl.write(data) + size += len(data) + if callback is not None: + callback(size, file_size) + if len(data) == 0: + break + finally: + fr.close() + return size + + def get(self, remotepath, localpath, callback=None): + """ + Copy a remote file (C{remotepath}) from the SFTP server to the local + host as C{localpath}. Any exception raised by operations will be + passed through. This method is primarily provided as a convenience. + + @param remotepath: the remote file to copy + @type remotepath: str + @param localpath: the destination path on the local host + @type localpath: str + @param callback: optional callback function that accepts the bytes + transferred so far and the total bytes to be transferred + (since 1.7.4) + @type callback: function(int, int) + + @since: 1.4 + """ + file_size = self.stat(remotepath).st_size + fl = file(localpath, 'wb') + try: + size = self.getfo(remotepath, fl, callback) + finally: + fl.close() + s = os.stat(localpath) + if s.st_size != size: + raise IOError('size mismatch in get! %d != %d' % (s.st_size, size)) + + + ### internals... + + + def _request(self, t, *arg): + num = self._async_request(type(None), t, *arg) + return self._read_response(num) + + def _async_request(self, fileobj, t, *arg): + # this method may be called from other threads (prefetch) + self._lock.acquire() + try: + msg = Message() + msg.add_int(self.request_number) + for item in arg: + if isinstance(item, int): + msg.add_int(item) + elif isinstance(item, long): + msg.add_int64(item) + elif isinstance(item, str): + msg.add_string(item) + elif isinstance(item, SFTPAttributes): + item._pack(msg) + else: + raise Exception('unknown type for %r type %r' % (item, type(item))) + num = self.request_number + self._expecting[num] = fileobj + self._send_packet(t, str(msg)) + self.request_number += 1 + finally: + self._lock.release() + return num + + def _read_response(self, waitfor=None): + while True: + try: + t, data = self._read_packet() + except EOFError, e: + raise SSHException('Server connection dropped: %s' % (str(e),)) + msg = Message(data) + num = msg.get_int() + if num not in self._expecting: + # might be response for a file that was closed before responses came back + self._log(DEBUG, 'Unexpected response #%d' % (num,)) + if waitfor is None: + # just doing a single check + break + continue + fileobj = self._expecting[num] + del self._expecting[num] + if num == waitfor: + # synchronous + if t == CMD_STATUS: + self._convert_status(msg) + return t, msg + if fileobj is not type(None): + fileobj._async_response(t, msg) + if waitfor is None: + # just doing a single check + break + return (None, None) + + def _finish_responses(self, fileobj): + while fileobj in self._expecting.values(): + self._read_response() + fileobj._check_exception() + + def _convert_status(self, msg): + """ + Raises EOFError or IOError on error status; otherwise does nothing. + """ + code = msg.get_int() + text = msg.get_string() + if code == SFTP_OK: + return + elif code == SFTP_EOF: + raise EOFError(text) + elif code == SFTP_NO_SUCH_FILE: + # clever idea from john a. meinel: map the error codes to errno + raise IOError(errno.ENOENT, text) + elif code == SFTP_PERMISSION_DENIED: + raise IOError(errno.EACCES, text) + else: + raise IOError(text) + + def _adjust_cwd(self, path): + """ + Return an adjusted path if we're emulating a "current working + directory" for the server. + """ + if type(path) is unicode: + path = path.encode('utf-8') + if self._cwd is None: + return path + if (len(path) > 0) and (path[0] == '/'): + # absolute path + return path + if self._cwd == '/': + return self._cwd + path + return self._cwd + '/' + path + + +class SFTP (SFTPClient): + "an alias for L{SFTPClient} for backwards compatability" + pass diff --git a/contrib/site-packages/paramiko/sftp_file.py b/contrib/site-packages/paramiko/sftp_file.py new file mode 100644 index 00000000..4ec936d0 --- /dev/null +++ b/contrib/site-packages/paramiko/sftp_file.py @@ -0,0 +1,489 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +L{SFTPFile} +""" + +from binascii import hexlify +from collections import deque +import socket +import threading +import time + +from paramiko.common import * +from paramiko.sftp import * +from paramiko.file import BufferedFile +from paramiko.sftp_attr import SFTPAttributes + + +class SFTPFile (BufferedFile): + """ + Proxy object for a file on the remote server, in client mode SFTP. + + Instances of this class may be used as context managers in the same way + that built-in Python file objects are. + """ + + # Some sftp servers will choke if you send read/write requests larger than + # this size. + MAX_REQUEST_SIZE = 32768 + + def __init__(self, sftp, handle, mode='r', bufsize=-1): + BufferedFile.__init__(self) + self.sftp = sftp + self.handle = handle + BufferedFile._set_mode(self, mode, bufsize) + self.pipelined = False + self._prefetching = False + self._prefetch_done = False + self._prefetch_data = {} + self._prefetch_reads = [] + self._saved_exception = None + self._reqs = deque() + + def __del__(self): + self._close(async=True) + + def close(self): + self._close(async=False) + + def _close(self, async=False): + # We allow double-close without signaling an error, because real + # Python file objects do. However, we must protect against actually + # sending multiple CMD_CLOSE packets, because after we close our + # handle, the same handle may be re-allocated by the server, and we + # may end up mysteriously closing some random other file. (This is + # especially important because we unconditionally call close() from + # __del__.) + if self._closed: + return + self.sftp._log(DEBUG, 'close(%s)' % hexlify(self.handle)) + if self.pipelined: + self.sftp._finish_responses(self) + BufferedFile.close(self) + try: + if async: + # GC'd file handle could be called from an arbitrary thread -- don't wait for a response + self.sftp._async_request(type(None), CMD_CLOSE, self.handle) + else: + self.sftp._request(CMD_CLOSE, self.handle) + except EOFError: + # may have outlived the Transport connection + pass + except (IOError, socket.error): + # may have outlived the Transport connection + pass + + def _data_in_prefetch_requests(self, offset, size): + k = [i for i in self._prefetch_reads if i[0] <= offset] + if len(k) == 0: + return False + k.sort(lambda x, y: cmp(x[0], y[0])) + buf_offset, buf_size = k[-1] + if buf_offset + buf_size <= offset: + # prefetch request ends before this one begins + return False + if buf_offset + buf_size >= offset + size: + # inclusive + return True + # well, we have part of the request. see if another chunk has the rest. + return self._data_in_prefetch_requests(buf_offset + buf_size, offset + size - buf_offset - buf_size) + + def _data_in_prefetch_buffers(self, offset): + """ + if a block of data is present in the prefetch buffers, at the given + offset, return the offset of the relevant prefetch buffer. otherwise, + return None. this guarantees nothing about the number of bytes + collected in the prefetch buffer so far. + """ + k = [i for i in self._prefetch_data.keys() if i <= offset] + if len(k) == 0: + return None + index = max(k) + buf_offset = offset - index + if buf_offset >= len(self._prefetch_data[index]): + # it's not here + return None + return index + + def _read_prefetch(self, size): + """ + read data out of the prefetch buffer, if possible. if the data isn't + in the buffer, return None. otherwise, behaves like a normal read. + """ + # while not closed, and haven't fetched past the current position, and haven't reached EOF... + while True: + offset = self._data_in_prefetch_buffers(self._realpos) + if offset is not None: + break + if self._prefetch_done or self._closed: + break + self.sftp._read_response() + self._check_exception() + if offset is None: + self._prefetching = False + return None + prefetch = self._prefetch_data[offset] + del self._prefetch_data[offset] + + buf_offset = self._realpos - offset + if buf_offset > 0: + self._prefetch_data[offset] = prefetch[:buf_offset] + prefetch = prefetch[buf_offset:] + if size < len(prefetch): + self._prefetch_data[self._realpos + size] = prefetch[size:] + prefetch = prefetch[:size] + return prefetch + + def _read(self, size): + size = min(size, self.MAX_REQUEST_SIZE) + if self._prefetching: + data = self._read_prefetch(size) + if data is not None: + return data + t, msg = self.sftp._request(CMD_READ, self.handle, long(self._realpos), int(size)) + if t != CMD_DATA: + raise SFTPError('Expected data') + return msg.get_string() + + def _write(self, data): + # may write less than requested if it would exceed max packet size + chunk = min(len(data), self.MAX_REQUEST_SIZE) + self._reqs.append(self.sftp._async_request(type(None), CMD_WRITE, self.handle, long(self._realpos), str(data[:chunk]))) + if not self.pipelined or (len(self._reqs) > 100 and self.sftp.sock.recv_ready()): + while len(self._reqs): + req = self._reqs.popleft() + t, msg = self.sftp._read_response(req) + if t != CMD_STATUS: + raise SFTPError('Expected status') + # convert_status already called + return chunk + + def settimeout(self, timeout): + """ + Set a timeout on read/write operations on the underlying socket or + ssh L{Channel}. + + @see: L{Channel.settimeout} + @param timeout: seconds to wait for a pending read/write operation + before raising C{socket.timeout}, or C{None} for no timeout + @type timeout: float + """ + self.sftp.sock.settimeout(timeout) + + def gettimeout(self): + """ + Returns the timeout in seconds (as a float) associated with the socket + or ssh L{Channel} used for this file. + + @see: L{Channel.gettimeout} + @rtype: float + """ + return self.sftp.sock.gettimeout() + + def setblocking(self, blocking): + """ + Set blocking or non-blocking mode on the underiying socket or ssh + L{Channel}. + + @see: L{Channel.setblocking} + @param blocking: 0 to set non-blocking mode; non-0 to set blocking + mode. + @type blocking: int + """ + self.sftp.sock.setblocking(blocking) + + def seek(self, offset, whence=0): + self.flush() + if whence == self.SEEK_SET: + self._realpos = self._pos = offset + elif whence == self.SEEK_CUR: + self._pos += offset + self._realpos = self._pos + else: + self._realpos = self._pos = self._get_size() + offset + self._rbuffer = '' + + def stat(self): + """ + Retrieve information about this file from the remote system. This is + exactly like L{SFTP.stat}, except that it operates on an already-open + file. + + @return: an object containing attributes about this file. + @rtype: SFTPAttributes + """ + t, msg = self.sftp._request(CMD_FSTAT, self.handle) + if t != CMD_ATTRS: + raise SFTPError('Expected attributes') + return SFTPAttributes._from_msg(msg) + + def chmod(self, mode): + """ + Change the mode (permissions) of this file. The permissions are + unix-style and identical to those used by python's C{os.chmod} + function. + + @param mode: new permissions + @type mode: int + """ + self.sftp._log(DEBUG, 'chmod(%s, %r)' % (hexlify(self.handle), mode)) + attr = SFTPAttributes() + attr.st_mode = mode + self.sftp._request(CMD_FSETSTAT, self.handle, attr) + + def chown(self, uid, gid): + """ + Change the owner (C{uid}) and group (C{gid}) of this file. As with + python's C{os.chown} function, you must pass both arguments, so if you + only want to change one, use L{stat} first to retrieve the current + owner and group. + + @param uid: new owner's uid + @type uid: int + @param gid: new group id + @type gid: int + """ + self.sftp._log(DEBUG, 'chown(%s, %r, %r)' % (hexlify(self.handle), uid, gid)) + attr = SFTPAttributes() + attr.st_uid, attr.st_gid = uid, gid + self.sftp._request(CMD_FSETSTAT, self.handle, attr) + + def utime(self, times): + """ + Set the access and modified times of this file. If + C{times} is C{None}, then the file's access and modified times are set + to the current time. Otherwise, C{times} must be a 2-tuple of numbers, + of the form C{(atime, mtime)}, which is used to set the access and + modified times, respectively. This bizarre API is mimicked from python + for the sake of consistency -- I apologize. + + @param times: C{None} or a tuple of (access time, modified time) in + standard internet epoch time (seconds since 01 January 1970 GMT) + @type times: tuple(int) + """ + if times is None: + times = (time.time(), time.time()) + self.sftp._log(DEBUG, 'utime(%s, %r)' % (hexlify(self.handle), times)) + attr = SFTPAttributes() + attr.st_atime, attr.st_mtime = times + self.sftp._request(CMD_FSETSTAT, self.handle, attr) + + def truncate(self, size): + """ + Change the size of this file. This usually extends + or shrinks the size of the file, just like the C{truncate()} method on + python file objects. + + @param size: the new size of the file + @type size: int or long + """ + self.sftp._log(DEBUG, 'truncate(%s, %r)' % (hexlify(self.handle), size)) + attr = SFTPAttributes() + attr.st_size = size + self.sftp._request(CMD_FSETSTAT, self.handle, attr) + + def check(self, hash_algorithm, offset=0, length=0, block_size=0): + """ + Ask the server for a hash of a section of this file. This can be used + to verify a successful upload or download, or for various rsync-like + operations. + + The file is hashed from C{offset}, for C{length} bytes. If C{length} + is 0, the remainder of the file is hashed. Thus, if both C{offset} + and C{length} are zero, the entire file is hashed. + + Normally, C{block_size} will be 0 (the default), and this method will + return a byte string representing the requested hash (for example, a + string of length 16 for MD5, or 20 for SHA-1). If a non-zero + C{block_size} is given, each chunk of the file (from C{offset} to + C{offset + length}) of C{block_size} bytes is computed as a separate + hash. The hash results are all concatenated and returned as a single + string. + + For example, C{check('sha1', 0, 1024, 512)} will return a string of + length 40. The first 20 bytes will be the SHA-1 of the first 512 bytes + of the file, and the last 20 bytes will be the SHA-1 of the next 512 + bytes. + + @param hash_algorithm: the name of the hash algorithm to use (normally + C{"sha1"} or C{"md5"}) + @type hash_algorithm: str + @param offset: offset into the file to begin hashing (0 means to start + from the beginning) + @type offset: int or long + @param length: number of bytes to hash (0 means continue to the end of + the file) + @type length: int or long + @param block_size: number of bytes to hash per result (must not be less + than 256; 0 means to compute only one hash of the entire segment) + @type block_size: int + @return: string of bytes representing the hash of each block, + concatenated together + @rtype: str + + @note: Many (most?) servers don't support this extension yet. + + @raise IOError: if the server doesn't support the "check-file" + extension, or possibly doesn't support the hash algorithm + requested + + @since: 1.4 + """ + t, msg = self.sftp._request(CMD_EXTENDED, 'check-file', self.handle, + hash_algorithm, long(offset), long(length), block_size) + ext = msg.get_string() + alg = msg.get_string() + data = msg.get_remainder() + return data + + def set_pipelined(self, pipelined=True): + """ + Turn on/off the pipelining of write operations to this file. When + pipelining is on, paramiko won't wait for the server response after + each write operation. Instead, they're collected as they come in. + At the first non-write operation (including L{close}), all remaining + server responses are collected. This means that if there was an error + with one of your later writes, an exception might be thrown from + within L{close} instead of L{write}. + + By default, files are I{not} pipelined. + + @param pipelined: C{True} if pipelining should be turned on for this + file; C{False} otherwise + @type pipelined: bool + + @since: 1.5 + """ + self.pipelined = pipelined + + def prefetch(self): + """ + Pre-fetch the remaining contents of this file in anticipation of + future L{read} calls. If reading the entire file, pre-fetching can + dramatically improve the download speed by avoiding roundtrip latency. + The file's contents are incrementally buffered in a background thread. + + The prefetched data is stored in a buffer until read via the L{read} + method. Once data has been read, it's removed from the buffer. The + data may be read in a random order (using L{seek}); chunks of the + buffer that haven't been read will continue to be buffered. + + @since: 1.5.1 + """ + size = self.stat().st_size + # queue up async reads for the rest of the file + chunks = [] + n = self._realpos + while n < size: + chunk = min(self.MAX_REQUEST_SIZE, size - n) + chunks.append((n, chunk)) + n += chunk + if len(chunks) > 0: + self._start_prefetch(chunks) + + def readv(self, chunks): + """ + Read a set of blocks from the file by (offset, length). This is more + efficient than doing a series of L{seek} and L{read} calls, since the + prefetch machinery is used to retrieve all the requested blocks at + once. + + @param chunks: a list of (offset, length) tuples indicating which + sections of the file to read + @type chunks: list(tuple(long, int)) + @return: a list of blocks read, in the same order as in C{chunks} + @rtype: list(str) + + @since: 1.5.4 + """ + self.sftp._log(DEBUG, 'readv(%s, %r)' % (hexlify(self.handle), chunks)) + + read_chunks = [] + for offset, size in chunks: + # don't fetch data that's already in the prefetch buffer + if self._data_in_prefetch_buffers(offset) or self._data_in_prefetch_requests(offset, size): + continue + + # break up anything larger than the max read size + while size > 0: + chunk_size = min(size, self.MAX_REQUEST_SIZE) + read_chunks.append((offset, chunk_size)) + offset += chunk_size + size -= chunk_size + + self._start_prefetch(read_chunks) + # now we can just devolve to a bunch of read()s :) + for x in chunks: + self.seek(x[0]) + yield self.read(x[1]) + + + ### internals... + + + def _get_size(self): + try: + return self.stat().st_size + except: + return 0 + + def _start_prefetch(self, chunks): + self._prefetching = True + self._prefetch_done = False + self._prefetch_reads.extend(chunks) + + t = threading.Thread(target=self._prefetch_thread, args=(chunks,)) + t.setDaemon(True) + t.start() + + def _prefetch_thread(self, chunks): + # do these read requests in a temporary thread because there may be + # a lot of them, so it may block. + for offset, length in chunks: + self.sftp._async_request(self, CMD_READ, self.handle, long(offset), int(length)) + + def _async_response(self, t, msg): + if t == CMD_STATUS: + # save exception and re-raise it on next file operation + try: + self.sftp._convert_status(msg) + except Exception, x: + self._saved_exception = x + return + if t != CMD_DATA: + raise SFTPError('Expected data') + data = msg.get_string() + offset, length = self._prefetch_reads.pop(0) + self._prefetch_data[offset] = data + if len(self._prefetch_reads) == 0: + self._prefetch_done = True + + def _check_exception(self): + "if there's a saved exception, raise & clear it" + if self._saved_exception is not None: + x = self._saved_exception + self._saved_exception = None + raise x + + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + self.close() diff --git a/contrib/site-packages/paramiko/sftp_handle.py b/contrib/site-packages/paramiko/sftp_handle.py new file mode 100644 index 00000000..29d3d0d8 --- /dev/null +++ b/contrib/site-packages/paramiko/sftp_handle.py @@ -0,0 +1,202 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +Abstraction of an SFTP file handle (for server mode). +""" + +import os + +from paramiko.common import * +from paramiko.sftp import * + + +class SFTPHandle (object): + """ + Abstract object representing a handle to an open file (or folder) in an + SFTP server implementation. Each handle has a string representation used + by the client to refer to the underlying file. + + Server implementations can (and should) subclass SFTPHandle to implement + features of a file handle, like L{stat} or L{chattr}. + """ + def __init__(self, flags=0): + """ + Create a new file handle representing a local file being served over + SFTP. If C{flags} is passed in, it's used to determine if the file + is open in append mode. + + @param flags: optional flags as passed to L{SFTPServerInterface.open} + @type flags: int + """ + self.__flags = flags + self.__name = None + # only for handles to folders: + self.__files = { } + self.__tell = None + + def close(self): + """ + When a client closes a file, this method is called on the handle. + Normally you would use this method to close the underlying OS level + file object(s). + + The default implementation checks for attributes on C{self} named + C{readfile} and/or C{writefile}, and if either or both are present, + their C{close()} methods are called. This means that if you are + using the default implementations of L{read} and L{write}, this + method's default implementation should be fine also. + """ + readfile = getattr(self, 'readfile', None) + if readfile is not None: + readfile.close() + writefile = getattr(self, 'writefile', None) + if writefile is not None: + writefile.close() + + def read(self, offset, length): + """ + Read up to C{length} bytes from this file, starting at position + C{offset}. The offset may be a python long, since SFTP allows it + to be 64 bits. + + If the end of the file has been reached, this method may return an + empty string to signify EOF, or it may also return L{SFTP_EOF}. + + The default implementation checks for an attribute on C{self} named + C{readfile}, and if present, performs the read operation on the python + file-like object found there. (This is meant as a time saver for the + common case where you are wrapping a python file object.) + + @param offset: position in the file to start reading from. + @type offset: int or long + @param length: number of bytes to attempt to read. + @type length: int + @return: data read from the file, or an SFTP error code. + @rtype: str + """ + readfile = getattr(self, 'readfile', None) + if readfile is None: + return SFTP_OP_UNSUPPORTED + try: + if self.__tell is None: + self.__tell = readfile.tell() + if offset != self.__tell: + readfile.seek(offset) + self.__tell = offset + data = readfile.read(length) + except IOError, e: + self.__tell = None + return SFTPServer.convert_errno(e.errno) + self.__tell += len(data) + return data + + def write(self, offset, data): + """ + Write C{data} into this file at position C{offset}. Extending the + file past its original end is expected. Unlike python's normal + C{write()} methods, this method cannot do a partial write: it must + write all of C{data} or else return an error. + + The default implementation checks for an attribute on C{self} named + C{writefile}, and if present, performs the write operation on the + python file-like object found there. The attribute is named + differently from C{readfile} to make it easy to implement read-only + (or write-only) files, but if both attributes are present, they should + refer to the same file. + + @param offset: position in the file to start reading from. + @type offset: int or long + @param data: data to write into the file. + @type data: str + @return: an SFTP error code like L{SFTP_OK}. + """ + writefile = getattr(self, 'writefile', None) + if writefile is None: + return SFTP_OP_UNSUPPORTED + try: + # in append mode, don't care about seeking + if (self.__flags & os.O_APPEND) == 0: + if self.__tell is None: + self.__tell = writefile.tell() + if offset != self.__tell: + writefile.seek(offset) + self.__tell = offset + writefile.write(data) + writefile.flush() + except IOError, e: + self.__tell = None + return SFTPServer.convert_errno(e.errno) + if self.__tell is not None: + self.__tell += len(data) + return SFTP_OK + + def stat(self): + """ + Return an L{SFTPAttributes} object referring to this open file, or an + error code. This is equivalent to L{SFTPServerInterface.stat}, except + it's called on an open file instead of a path. + + @return: an attributes object for the given file, or an SFTP error + code (like L{SFTP_PERMISSION_DENIED}). + @rtype: L{SFTPAttributes} I{or error code} + """ + return SFTP_OP_UNSUPPORTED + + def chattr(self, attr): + """ + Change the attributes of this file. The C{attr} object will contain + only those fields provided by the client in its request, so you should + check for the presence of fields before using them. + + @param attr: the attributes to change on this file. + @type attr: L{SFTPAttributes} + @return: an error code like L{SFTP_OK}. + @rtype: int + """ + return SFTP_OP_UNSUPPORTED + + + ### internals... + + + def _set_files(self, files): + """ + Used by the SFTP server code to cache a directory listing. (In + the SFTP protocol, listing a directory is a multi-stage process + requiring a temporary handle.) + """ + self.__files = files + + def _get_next_files(self): + """ + Used by the SFTP server code to retreive a cached directory + listing. + """ + fnlist = self.__files[:16] + self.__files = self.__files[16:] + return fnlist + + def _get_name(self): + return self.__name + + def _set_name(self, name): + self.__name = name + + +from paramiko.sftp_server import SFTPServer diff --git a/contrib/site-packages/paramiko/sftp_server.py b/contrib/site-packages/paramiko/sftp_server.py new file mode 100644 index 00000000..1833c2e8 --- /dev/null +++ b/contrib/site-packages/paramiko/sftp_server.py @@ -0,0 +1,444 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +Server-mode SFTP support. +""" + +import os +import errno + +from Crypto.Hash import MD5, SHA +from paramiko.common import * +from paramiko.server import SubsystemHandler +from paramiko.sftp import * +from paramiko.sftp_si import * +from paramiko.sftp_attr import * + + +# known hash algorithms for the "check-file" extension +_hash_class = { + 'sha1': SHA, + 'md5': MD5, +} + + +class SFTPServer (BaseSFTP, SubsystemHandler): + """ + Server-side SFTP subsystem support. Since this is a L{SubsystemHandler}, + it can be (and is meant to be) set as the handler for C{"sftp"} requests. + Use L{Transport.set_subsystem_handler} to activate this class. + """ + + def __init__(self, channel, name, server, sftp_si=SFTPServerInterface, *largs, **kwargs): + """ + The constructor for SFTPServer is meant to be called from within the + L{Transport} as a subsystem handler. C{server} and any additional + parameters or keyword parameters are passed from the original call to + L{Transport.set_subsystem_handler}. + + @param channel: channel passed from the L{Transport}. + @type channel: L{Channel} + @param name: name of the requested subsystem. + @type name: str + @param server: the server object associated with this channel and + subsystem + @type server: L{ServerInterface} + @param sftp_si: a subclass of L{SFTPServerInterface} to use for handling + individual requests. + @type sftp_si: class + """ + BaseSFTP.__init__(self) + SubsystemHandler.__init__(self, channel, name, server) + transport = channel.get_transport() + self.logger = util.get_logger(transport.get_log_channel() + '.sftp') + self.ultra_debug = transport.get_hexdump() + self.next_handle = 1 + # map of handle-string to SFTPHandle for files & folders: + self.file_table = { } + self.folder_table = { } + self.server = sftp_si(server, *largs, **kwargs) + + def _log(self, level, msg): + if issubclass(type(msg), list): + for m in msg: + super(SFTPServer, self)._log(level, "[chan " + self.sock.get_name() + "] " + m) + else: + super(SFTPServer, self)._log(level, "[chan " + self.sock.get_name() + "] " + msg) + + def start_subsystem(self, name, transport, channel): + self.sock = channel + self._log(DEBUG, 'Started sftp server on channel %s' % repr(channel)) + self._send_server_version() + self.server.session_started() + while True: + try: + t, data = self._read_packet() + except EOFError: + self._log(DEBUG, 'EOF -- end of session') + return + except Exception, e: + self._log(DEBUG, 'Exception on channel: ' + str(e)) + self._log(DEBUG, util.tb_strings()) + return + msg = Message(data) + request_number = msg.get_int() + try: + self._process(t, request_number, msg) + except Exception, e: + self._log(DEBUG, 'Exception in server processing: ' + str(e)) + self._log(DEBUG, util.tb_strings()) + # send some kind of failure message, at least + try: + self._send_status(request_number, SFTP_FAILURE) + except: + pass + + def finish_subsystem(self): + self.server.session_ended() + super(SFTPServer, self).finish_subsystem() + # close any file handles that were left open (so we can return them to the OS quickly) + for f in self.file_table.itervalues(): + f.close() + for f in self.folder_table.itervalues(): + f.close() + self.file_table = {} + self.folder_table = {} + + def convert_errno(e): + """ + Convert an errno value (as from an C{OSError} or C{IOError}) into a + standard SFTP result code. This is a convenience function for trapping + exceptions in server code and returning an appropriate result. + + @param e: an errno code, as from C{OSError.errno}. + @type e: int + @return: an SFTP error code like L{SFTP_NO_SUCH_FILE}. + @rtype: int + """ + if e == errno.EACCES: + # permission denied + return SFTP_PERMISSION_DENIED + elif (e == errno.ENOENT) or (e == errno.ENOTDIR): + # no such file + return SFTP_NO_SUCH_FILE + else: + return SFTP_FAILURE + convert_errno = staticmethod(convert_errno) + + def set_file_attr(filename, attr): + """ + Change a file's attributes on the local filesystem. The contents of + C{attr} are used to change the permissions, owner, group ownership, + and/or modification & access time of the file, depending on which + attributes are present in C{attr}. + + This is meant to be a handy helper function for translating SFTP file + requests into local file operations. + + @param filename: name of the file to alter (should usually be an + absolute path). + @type filename: str + @param attr: attributes to change. + @type attr: L{SFTPAttributes} + """ + if sys.platform != 'win32': + # mode operations are meaningless on win32 + if attr._flags & attr.FLAG_PERMISSIONS: + os.chmod(filename, attr.st_mode) + if attr._flags & attr.FLAG_UIDGID: + os.chown(filename, attr.st_uid, attr.st_gid) + if attr._flags & attr.FLAG_AMTIME: + os.utime(filename, (attr.st_atime, attr.st_mtime)) + if attr._flags & attr.FLAG_SIZE: + open(filename, 'w+').truncate(attr.st_size) + set_file_attr = staticmethod(set_file_attr) + + + ### internals... + + + def _response(self, request_number, t, *arg): + msg = Message() + msg.add_int(request_number) + for item in arg: + if type(item) is int: + msg.add_int(item) + elif type(item) is long: + msg.add_int64(item) + elif type(item) is str: + msg.add_string(item) + elif type(item) is SFTPAttributes: + item._pack(msg) + else: + raise Exception('unknown type for ' + repr(item) + ' type ' + repr(type(item))) + self._send_packet(t, str(msg)) + + def _send_handle_response(self, request_number, handle, folder=False): + if not issubclass(type(handle), SFTPHandle): + # must be error code + self._send_status(request_number, handle) + return + handle._set_name('hx%d' % self.next_handle) + self.next_handle += 1 + if folder: + self.folder_table[handle._get_name()] = handle + else: + self.file_table[handle._get_name()] = handle + self._response(request_number, CMD_HANDLE, handle._get_name()) + + def _send_status(self, request_number, code, desc=None): + if desc is None: + try: + desc = SFTP_DESC[code] + except IndexError: + desc = 'Unknown' + # some clients expect a "langauge" tag at the end (but don't mind it being blank) + self._response(request_number, CMD_STATUS, code, desc, '') + + def _open_folder(self, request_number, path): + resp = self.server.list_folder(path) + if issubclass(type(resp), list): + # got an actual list of filenames in the folder + folder = SFTPHandle() + folder._set_files(resp) + self._send_handle_response(request_number, folder, True) + return + # must be an error code + self._send_status(request_number, resp) + + def _read_folder(self, request_number, folder): + flist = folder._get_next_files() + if len(flist) == 0: + self._send_status(request_number, SFTP_EOF) + return + msg = Message() + msg.add_int(request_number) + msg.add_int(len(flist)) + for attr in flist: + msg.add_string(attr.filename) + msg.add_string(str(attr)) + attr._pack(msg) + self._send_packet(CMD_NAME, str(msg)) + + def _check_file(self, request_number, msg): + # this extension actually comes from v6 protocol, but since it's an + # extension, i feel like we can reasonably support it backported. + # it's very useful for verifying uploaded files or checking for + # rsync-like differences between local and remote files. + handle = msg.get_string() + alg_list = msg.get_list() + start = msg.get_int64() + length = msg.get_int64() + block_size = msg.get_int() + if handle not in self.file_table: + self._send_status(request_number, SFTP_BAD_MESSAGE, 'Invalid handle') + return + f = self.file_table[handle] + for x in alg_list: + if x in _hash_class: + algname = x + alg = _hash_class[x] + break + else: + self._send_status(request_number, SFTP_FAILURE, 'No supported hash types found') + return + if length == 0: + st = f.stat() + if not issubclass(type(st), SFTPAttributes): + self._send_status(request_number, st, 'Unable to stat file') + return + length = st.st_size - start + if block_size == 0: + block_size = length + if block_size < 256: + self._send_status(request_number, SFTP_FAILURE, 'Block size too small') + return + + sum_out = '' + offset = start + while offset < start + length: + blocklen = min(block_size, start + length - offset) + # don't try to read more than about 64KB at a time + chunklen = min(blocklen, 65536) + count = 0 + hash_obj = alg.new() + while count < blocklen: + data = f.read(offset, chunklen) + if not type(data) is str: + self._send_status(request_number, data, 'Unable to hash file') + return + hash_obj.update(data) + count += len(data) + offset += count + sum_out += hash_obj.digest() + + msg = Message() + msg.add_int(request_number) + msg.add_string('check-file') + msg.add_string(algname) + msg.add_bytes(sum_out) + self._send_packet(CMD_EXTENDED_REPLY, str(msg)) + + def _convert_pflags(self, pflags): + "convert SFTP-style open() flags to python's os.open() flags" + if (pflags & SFTP_FLAG_READ) and (pflags & SFTP_FLAG_WRITE): + flags = os.O_RDWR + elif pflags & SFTP_FLAG_WRITE: + flags = os.O_WRONLY + else: + flags = os.O_RDONLY + if pflags & SFTP_FLAG_APPEND: + flags |= os.O_APPEND + if pflags & SFTP_FLAG_CREATE: + flags |= os.O_CREAT + if pflags & SFTP_FLAG_TRUNC: + flags |= os.O_TRUNC + if pflags & SFTP_FLAG_EXCL: + flags |= os.O_EXCL + return flags + + def _process(self, t, request_number, msg): + self._log(DEBUG, 'Request: %s' % CMD_NAMES[t]) + if t == CMD_OPEN: + path = msg.get_string() + flags = self._convert_pflags(msg.get_int()) + attr = SFTPAttributes._from_msg(msg) + self._send_handle_response(request_number, self.server.open(path, flags, attr)) + elif t == CMD_CLOSE: + handle = msg.get_string() + if handle in self.folder_table: + del self.folder_table[handle] + self._send_status(request_number, SFTP_OK) + return + if handle in self.file_table: + self.file_table[handle].close() + del self.file_table[handle] + self._send_status(request_number, SFTP_OK) + return + self._send_status(request_number, SFTP_BAD_MESSAGE, 'Invalid handle') + elif t == CMD_READ: + handle = msg.get_string() + offset = msg.get_int64() + length = msg.get_int() + if handle not in self.file_table: + self._send_status(request_number, SFTP_BAD_MESSAGE, 'Invalid handle') + return + data = self.file_table[handle].read(offset, length) + if type(data) is str: + if len(data) == 0: + self._send_status(request_number, SFTP_EOF) + else: + self._response(request_number, CMD_DATA, data) + else: + self._send_status(request_number, data) + elif t == CMD_WRITE: + handle = msg.get_string() + offset = msg.get_int64() + data = msg.get_string() + if handle not in self.file_table: + self._send_status(request_number, SFTP_BAD_MESSAGE, 'Invalid handle') + return + self._send_status(request_number, self.file_table[handle].write(offset, data)) + elif t == CMD_REMOVE: + path = msg.get_string() + self._send_status(request_number, self.server.remove(path)) + elif t == CMD_RENAME: + oldpath = msg.get_string() + newpath = msg.get_string() + self._send_status(request_number, self.server.rename(oldpath, newpath)) + elif t == CMD_MKDIR: + path = msg.get_string() + attr = SFTPAttributes._from_msg(msg) + self._send_status(request_number, self.server.mkdir(path, attr)) + elif t == CMD_RMDIR: + path = msg.get_string() + self._send_status(request_number, self.server.rmdir(path)) + elif t == CMD_OPENDIR: + path = msg.get_string() + self._open_folder(request_number, path) + return + elif t == CMD_READDIR: + handle = msg.get_string() + if handle not in self.folder_table: + self._send_status(request_number, SFTP_BAD_MESSAGE, 'Invalid handle') + return + folder = self.folder_table[handle] + self._read_folder(request_number, folder) + elif t == CMD_STAT: + path = msg.get_string() + resp = self.server.stat(path) + if issubclass(type(resp), SFTPAttributes): + self._response(request_number, CMD_ATTRS, resp) + else: + self._send_status(request_number, resp) + elif t == CMD_LSTAT: + path = msg.get_string() + resp = self.server.lstat(path) + if issubclass(type(resp), SFTPAttributes): + self._response(request_number, CMD_ATTRS, resp) + else: + self._send_status(request_number, resp) + elif t == CMD_FSTAT: + handle = msg.get_string() + if handle not in self.file_table: + self._send_status(request_number, SFTP_BAD_MESSAGE, 'Invalid handle') + return + resp = self.file_table[handle].stat() + if issubclass(type(resp), SFTPAttributes): + self._response(request_number, CMD_ATTRS, resp) + else: + self._send_status(request_number, resp) + elif t == CMD_SETSTAT: + path = msg.get_string() + attr = SFTPAttributes._from_msg(msg) + self._send_status(request_number, self.server.chattr(path, attr)) + elif t == CMD_FSETSTAT: + handle = msg.get_string() + attr = SFTPAttributes._from_msg(msg) + if handle not in self.file_table: + self._response(request_number, SFTP_BAD_MESSAGE, 'Invalid handle') + return + self._send_status(request_number, self.file_table[handle].chattr(attr)) + elif t == CMD_READLINK: + path = msg.get_string() + resp = self.server.readlink(path) + if type(resp) is str: + self._response(request_number, CMD_NAME, 1, resp, '', SFTPAttributes()) + else: + self._send_status(request_number, resp) + elif t == CMD_SYMLINK: + # the sftp 2 draft is incorrect here! path always follows target_path + target_path = msg.get_string() + path = msg.get_string() + self._send_status(request_number, self.server.symlink(target_path, path)) + elif t == CMD_REALPATH: + path = msg.get_string() + rpath = self.server.canonicalize(path) + self._response(request_number, CMD_NAME, 1, rpath, '', SFTPAttributes()) + elif t == CMD_EXTENDED: + tag = msg.get_string() + if tag == 'check-file': + self._check_file(request_number, msg) + else: + self._send_status(request_number, SFTP_OP_UNSUPPORTED) + else: + self._send_status(request_number, SFTP_OP_UNSUPPORTED) + + +from paramiko.sftp_handle import SFTPHandle diff --git a/contrib/site-packages/paramiko/sftp_si.py b/contrib/site-packages/paramiko/sftp_si.py new file mode 100644 index 00000000..b0ee3c42 --- /dev/null +++ b/contrib/site-packages/paramiko/sftp_si.py @@ -0,0 +1,310 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +L{SFTPServerInterface} is an interface to override for SFTP server support. +""" + +import os + +from paramiko.common import * +from paramiko.sftp import * + + +class SFTPServerInterface (object): + """ + This class defines an interface for controlling the behavior of paramiko + when using the L{SFTPServer} subsystem to provide an SFTP server. + + Methods on this class are called from the SFTP session's thread, so you can + block as long as necessary without affecting other sessions (even other + SFTP sessions). However, raising an exception will usually cause the SFTP + session to abruptly end, so you will usually want to catch exceptions and + return an appropriate error code. + + All paths are in string form instead of unicode because not all SFTP + clients & servers obey the requirement that paths be encoded in UTF-8. + """ + + def __init__ (self, server, *largs, **kwargs): + """ + Create a new SFTPServerInterface object. This method does nothing by + default and is meant to be overridden by subclasses. + + @param server: the server object associated with this channel and + SFTP subsystem + @type server: L{ServerInterface} + """ + super(SFTPServerInterface, self).__init__(*largs, **kwargs) + + def session_started(self): + """ + The SFTP server session has just started. This method is meant to be + overridden to perform any necessary setup before handling callbacks + from SFTP operations. + """ + pass + + def session_ended(self): + """ + The SFTP server session has just ended, either cleanly or via an + exception. This method is meant to be overridden to perform any + necessary cleanup before this C{SFTPServerInterface} object is + destroyed. + """ + pass + + def open(self, path, flags, attr): + """ + Open a file on the server and create a handle for future operations + on that file. On success, a new object subclassed from L{SFTPHandle} + should be returned. This handle will be used for future operations + on the file (read, write, etc). On failure, an error code such as + L{SFTP_PERMISSION_DENIED} should be returned. + + C{flags} contains the requested mode for opening (read-only, + write-append, etc) as a bitset of flags from the C{os} module: + - C{os.O_RDONLY} + - C{os.O_WRONLY} + - C{os.O_RDWR} + - C{os.O_APPEND} + - C{os.O_CREAT} + - C{os.O_TRUNC} + - C{os.O_EXCL} + (One of C{os.O_RDONLY}, C{os.O_WRONLY}, or C{os.O_RDWR} will always + be set.) + + The C{attr} object contains requested attributes of the file if it + has to be created. Some or all attribute fields may be missing if + the client didn't specify them. + + @note: The SFTP protocol defines all files to be in "binary" mode. + There is no equivalent to python's "text" mode. + + @param path: the requested path (relative or absolute) of the file + to be opened. + @type path: str + @param flags: flags or'd together from the C{os} module indicating the + requested mode for opening the file. + @type flags: int + @param attr: requested attributes of the file if it is newly created. + @type attr: L{SFTPAttributes} + @return: a new L{SFTPHandle} I{or error code}. + @rtype L{SFTPHandle} + """ + return SFTP_OP_UNSUPPORTED + + def list_folder(self, path): + """ + Return a list of files within a given folder. The C{path} will use + posix notation (C{"/"} separates folder names) and may be an absolute + or relative path. + + The list of files is expected to be a list of L{SFTPAttributes} + objects, which are similar in structure to the objects returned by + C{os.stat}. In addition, each object should have its C{filename} + field filled in, since this is important to a directory listing and + not normally present in C{os.stat} results. The method + L{SFTPAttributes.from_stat} will usually do what you want. + + In case of an error, you should return one of the C{SFTP_*} error + codes, such as L{SFTP_PERMISSION_DENIED}. + + @param path: the requested path (relative or absolute) to be listed. + @type path: str + @return: a list of the files in the given folder, using + L{SFTPAttributes} objects. + @rtype: list of L{SFTPAttributes} I{or error code} + + @note: You should normalize the given C{path} first (see the + C{os.path} module) and check appropriate permissions before returning + the list of files. Be careful of malicious clients attempting to use + relative paths to escape restricted folders, if you're doing a direct + translation from the SFTP server path to your local filesystem. + """ + return SFTP_OP_UNSUPPORTED + + def stat(self, path): + """ + Return an L{SFTPAttributes} object for a path on the server, or an + error code. If your server supports symbolic links (also known as + "aliases"), you should follow them. (L{lstat} is the corresponding + call that doesn't follow symlinks/aliases.) + + @param path: the requested path (relative or absolute) to fetch + file statistics for. + @type path: str + @return: an attributes object for the given file, or an SFTP error + code (like L{SFTP_PERMISSION_DENIED}). + @rtype: L{SFTPAttributes} I{or error code} + """ + return SFTP_OP_UNSUPPORTED + + def lstat(self, path): + """ + Return an L{SFTPAttributes} object for a path on the server, or an + error code. If your server supports symbolic links (also known as + "aliases"), you should I{not} follow them -- instead, you should + return data on the symlink or alias itself. (L{stat} is the + corresponding call that follows symlinks/aliases.) + + @param path: the requested path (relative or absolute) to fetch + file statistics for. + @type path: str + @return: an attributes object for the given file, or an SFTP error + code (like L{SFTP_PERMISSION_DENIED}). + @rtype: L{SFTPAttributes} I{or error code} + """ + return SFTP_OP_UNSUPPORTED + + def remove(self, path): + """ + Delete a file, if possible. + + @param path: the requested path (relative or absolute) of the file + to delete. + @type path: str + @return: an SFTP error code like L{SFTP_OK}. + @rtype: int + """ + return SFTP_OP_UNSUPPORTED + + def rename(self, oldpath, newpath): + """ + Rename (or move) a file. The SFTP specification implies that this + method can be used to move an existing file into a different folder, + and since there's no other (easy) way to move files via SFTP, it's + probably a good idea to implement "move" in this method too, even for + files that cross disk partition boundaries, if at all possible. + + @note: You should return an error if a file with the same name as + C{newpath} already exists. (The rename operation should be + non-desctructive.) + + @param oldpath: the requested path (relative or absolute) of the + existing file. + @type oldpath: str + @param newpath: the requested new path of the file. + @type newpath: str + @return: an SFTP error code like L{SFTP_OK}. + @rtype: int + """ + return SFTP_OP_UNSUPPORTED + + def mkdir(self, path, attr): + """ + Create a new directory with the given attributes. The C{attr} + object may be considered a "hint" and ignored. + + The C{attr} object will contain only those fields provided by the + client in its request, so you should use C{hasattr} to check for + the presense of fields before using them. In some cases, the C{attr} + object may be completely empty. + + @param path: requested path (relative or absolute) of the new + folder. + @type path: str + @param attr: requested attributes of the new folder. + @type attr: L{SFTPAttributes} + @return: an SFTP error code like L{SFTP_OK}. + @rtype: int + """ + return SFTP_OP_UNSUPPORTED + + def rmdir(self, path): + """ + Remove a directory if it exists. The C{path} should refer to an + existing, empty folder -- otherwise this method should return an + error. + + @param path: requested path (relative or absolute) of the folder + to remove. + @type path: str + @return: an SFTP error code like L{SFTP_OK}. + @rtype: int + """ + return SFTP_OP_UNSUPPORTED + + def chattr(self, path, attr): + """ + Change the attributes of a file. The C{attr} object will contain + only those fields provided by the client in its request, so you + should check for the presence of fields before using them. + + @param path: requested path (relative or absolute) of the file to + change. + @type path: str + @param attr: requested attributes to change on the file. + @type attr: L{SFTPAttributes} + @return: an error code like L{SFTP_OK}. + @rtype: int + """ + return SFTP_OP_UNSUPPORTED + + def canonicalize(self, path): + """ + Return the canonical form of a path on the server. For example, + if the server's home folder is C{/home/foo}, the path + C{"../betty"} would be canonicalized to C{"/home/betty"}. Note + the obvious security issues: if you're serving files only from a + specific folder, you probably don't want this method to reveal path + names outside that folder. + + You may find the python methods in C{os.path} useful, especially + C{os.path.normpath} and C{os.path.realpath}. + + The default implementation returns C{os.path.normpath('/' + path)}. + """ + if os.path.isabs(path): + out = os.path.normpath(path) + else: + out = os.path.normpath('/' + path) + if sys.platform == 'win32': + # on windows, normalize backslashes to sftp/posix format + out = out.replace('\\', '/') + return out + + def readlink(self, path): + """ + Return the target of a symbolic link (or shortcut) on the server. + If the specified path doesn't refer to a symbolic link, an error + should be returned. + + @param path: path (relative or absolute) of the symbolic link. + @type path: str + @return: the target path of the symbolic link, or an error code like + L{SFTP_NO_SUCH_FILE}. + @rtype: str I{or error code} + """ + return SFTP_OP_UNSUPPORTED + + def symlink(self, target_path, path): + """ + Create a symbolic link on the server, as new pathname C{path}, + with C{target_path} as the target of the link. + + @param target_path: path (relative or absolute) of the target for + this new symbolic link. + @type target_path: str + @param path: path (relative or absolute) of the symbolic link to + create. + @type path: str + @return: an error code like C{SFTP_OK}. + @rtype: int + """ + return SFTP_OP_UNSUPPORTED diff --git a/contrib/site-packages/paramiko/ssh_exception.py b/contrib/site-packages/paramiko/ssh_exception.py new file mode 100644 index 00000000..b502b563 --- /dev/null +++ b/contrib/site-packages/paramiko/ssh_exception.py @@ -0,0 +1,132 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +Exceptions defined by paramiko. +""" + + +class SSHException (Exception): + """ + Exception raised by failures in SSH2 protocol negotiation or logic errors. + """ + pass + + +class AuthenticationException (SSHException): + """ + Exception raised when authentication failed for some reason. It may be + possible to retry with different credentials. (Other classes specify more + specific reasons.) + + @since: 1.6 + """ + pass + + +class PasswordRequiredException (AuthenticationException): + """ + Exception raised when a password is needed to unlock a private key file. + """ + pass + + +class BadAuthenticationType (AuthenticationException): + """ + Exception raised when an authentication type (like password) is used, but + the server isn't allowing that type. (It may only allow public-key, for + example.) + + @ivar allowed_types: list of allowed authentication types provided by the + server (possible values are: C{"none"}, C{"password"}, and + C{"publickey"}). + @type allowed_types: list + + @since: 1.1 + """ + allowed_types = [] + + def __init__(self, explanation, types): + AuthenticationException.__init__(self, explanation) + self.allowed_types = types + + def __str__(self): + return SSHException.__str__(self) + ' (allowed_types=%r)' % self.allowed_types + + +class PartialAuthentication (AuthenticationException): + """ + An internal exception thrown in the case of partial authentication. + """ + allowed_types = [] + + def __init__(self, types): + AuthenticationException.__init__(self, 'partial authentication') + self.allowed_types = types + + +class ChannelException (SSHException): + """ + Exception raised when an attempt to open a new L{Channel} fails. + + @ivar code: the error code returned by the server + @type code: int + + @since: 1.6 + """ + def __init__(self, code, text): + SSHException.__init__(self, text) + self.code = code + + +class BadHostKeyException (SSHException): + """ + The host key given by the SSH server did not match what we were expecting. + + @ivar hostname: the hostname of the SSH server + @type hostname: str + @ivar key: the host key presented by the server + @type key: L{PKey} + @ivar expected_key: the host key expected + @type expected_key: L{PKey} + + @since: 1.6 + """ + def __init__(self, hostname, got_key, expected_key): + SSHException.__init__(self, 'Host key for server %s does not match!' % hostname) + self.hostname = hostname + self.key = got_key + self.expected_key = expected_key + + +class ProxyCommandFailure (SSHException): + """ + The "ProxyCommand" found in the .ssh/config file returned an error. + + @ivar command: The command line that is generating this exception. + @type command: str + @ivar error: The error captured from the proxy command output. + @type error: str + """ + def __init__(self, command, error): + SSHException.__init__(self, + '"ProxyCommand (%s)" returned non-zero exit status: %s' % ( + command, error + ) + ) + self.error = error diff --git a/contrib/site-packages/paramiko/transport.py b/contrib/site-packages/paramiko/transport.py new file mode 100644 index 00000000..3155d3f8 --- /dev/null +++ b/contrib/site-packages/paramiko/transport.py @@ -0,0 +1,2158 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +L{Transport} handles the core SSH2 protocol. +""" + +import os +import socket +import string +import struct +import sys +import threading +import time +import weakref + +import paramiko +from paramiko import util +from paramiko.auth_handler import AuthHandler +from paramiko.channel import Channel +from paramiko.common import * +from paramiko.compress import ZlibCompressor, ZlibDecompressor +from paramiko.dsskey import DSSKey +from paramiko.kex_gex import KexGex +from paramiko.kex_group1 import KexGroup1 +from paramiko.message import Message +from paramiko.packet import Packetizer, NeedRekeyException +from paramiko.primes import ModulusPack +from paramiko.rsakey import RSAKey +from paramiko.ecdsakey import ECDSAKey +from paramiko.server import ServerInterface +from paramiko.sftp_client import SFTPClient +from paramiko.ssh_exception import (SSHException, BadAuthenticationType, + ChannelException, ProxyCommandFailure) +from paramiko.util import retry_on_signal + +from Crypto import Random +from Crypto.Cipher import Blowfish, AES, DES3, ARC4 +from Crypto.Hash import SHA, MD5 +try: + from Crypto.Util import Counter +except ImportError: + from paramiko.util import Counter + + +# for thread cleanup +_active_threads = [] +def _join_lingering_threads(): + for thr in _active_threads: + thr.stop_thread() +import atexit +atexit.register(_join_lingering_threads) + + +class SecurityOptions (object): + """ + Simple object containing the security preferences of an ssh transport. + These are tuples of acceptable ciphers, digests, key types, and key + exchange algorithms, listed in order of preference. + + Changing the contents and/or order of these fields affects the underlying + L{Transport} (but only if you change them before starting the session). + If you try to add an algorithm that paramiko doesn't recognize, + C{ValueError} will be raised. If you try to assign something besides a + tuple to one of the fields, C{TypeError} will be raised. + """ + __slots__ = [ 'ciphers', 'digests', 'key_types', 'kex', 'compression', '_transport' ] + + def __init__(self, transport): + self._transport = transport + + def __repr__(self): + """ + Returns a string representation of this object, for debugging. + + @rtype: str + """ + return '' % repr(self._transport) + + def _get_ciphers(self): + return self._transport._preferred_ciphers + + def _get_digests(self): + return self._transport._preferred_macs + + def _get_key_types(self): + return self._transport._preferred_keys + + def _get_kex(self): + return self._transport._preferred_kex + + def _get_compression(self): + return self._transport._preferred_compression + + def _set(self, name, orig, x): + if type(x) is list: + x = tuple(x) + if type(x) is not tuple: + raise TypeError('expected tuple or list') + possible = getattr(self._transport, orig).keys() + forbidden = filter(lambda n: n not in possible, x) + if len(forbidden) > 0: + raise ValueError('unknown cipher') + setattr(self._transport, name, x) + + def _set_ciphers(self, x): + self._set('_preferred_ciphers', '_cipher_info', x) + + def _set_digests(self, x): + self._set('_preferred_macs', '_mac_info', x) + + def _set_key_types(self, x): + self._set('_preferred_keys', '_key_info', x) + + def _set_kex(self, x): + self._set('_preferred_kex', '_kex_info', x) + + def _set_compression(self, x): + self._set('_preferred_compression', '_compression_info', x) + + ciphers = property(_get_ciphers, _set_ciphers, None, + "Symmetric encryption ciphers") + digests = property(_get_digests, _set_digests, None, + "Digest (one-way hash) algorithms") + key_types = property(_get_key_types, _set_key_types, None, + "Public-key algorithms") + kex = property(_get_kex, _set_kex, None, "Key exchange algorithms") + compression = property(_get_compression, _set_compression, None, + "Compression algorithms") + + +class ChannelMap (object): + def __init__(self): + # (id -> Channel) + self._map = weakref.WeakValueDictionary() + self._lock = threading.Lock() + + def put(self, chanid, chan): + self._lock.acquire() + try: + self._map[chanid] = chan + finally: + self._lock.release() + + def get(self, chanid): + self._lock.acquire() + try: + return self._map.get(chanid, None) + finally: + self._lock.release() + + def delete(self, chanid): + self._lock.acquire() + try: + try: + del self._map[chanid] + except KeyError: + pass + finally: + self._lock.release() + + def values(self): + self._lock.acquire() + try: + return self._map.values() + finally: + self._lock.release() + + def __len__(self): + self._lock.acquire() + try: + return len(self._map) + finally: + self._lock.release() + + +class Transport (threading.Thread): + """ + An SSH Transport attaches to a stream (usually a socket), negotiates an + encrypted session, authenticates, and then creates stream tunnels, called + L{Channel}s, across the session. Multiple channels can be multiplexed + across a single session (and often are, in the case of port forwardings). + """ + + _PROTO_ID = '2.0' + _CLIENT_ID = 'paramiko_%s' % (paramiko.__version__) + + _preferred_ciphers = ( 'aes128-ctr', 'aes256-ctr', 'aes128-cbc', 'blowfish-cbc', 'aes256-cbc', '3des-cbc', + 'arcfour128', 'arcfour256' ) + _preferred_macs = ( 'hmac-sha1', 'hmac-md5', 'hmac-sha1-96', 'hmac-md5-96' ) + _preferred_keys = ( 'ssh-rsa', 'ssh-dss', 'ecdsa-sha2-nistp256' ) + _preferred_kex = ( 'diffie-hellman-group1-sha1', 'diffie-hellman-group-exchange-sha1' ) + _preferred_compression = ( 'none', ) + + _cipher_info = { + 'aes128-ctr': { 'class': AES, 'mode': AES.MODE_CTR, 'block-size': 16, 'key-size': 16 }, + 'aes256-ctr': { 'class': AES, 'mode': AES.MODE_CTR, 'block-size': 16, 'key-size': 32 }, + 'blowfish-cbc': { 'class': Blowfish, 'mode': Blowfish.MODE_CBC, 'block-size': 8, 'key-size': 16 }, + 'aes128-cbc': { 'class': AES, 'mode': AES.MODE_CBC, 'block-size': 16, 'key-size': 16 }, + 'aes256-cbc': { 'class': AES, 'mode': AES.MODE_CBC, 'block-size': 16, 'key-size': 32 }, + '3des-cbc': { 'class': DES3, 'mode': DES3.MODE_CBC, 'block-size': 8, 'key-size': 24 }, + 'arcfour128': { 'class': ARC4, 'mode': None, 'block-size': 8, 'key-size': 16 }, + 'arcfour256': { 'class': ARC4, 'mode': None, 'block-size': 8, 'key-size': 32 }, + } + + _mac_info = { + 'hmac-sha1': { 'class': SHA, 'size': 20 }, + 'hmac-sha1-96': { 'class': SHA, 'size': 12 }, + 'hmac-md5': { 'class': MD5, 'size': 16 }, + 'hmac-md5-96': { 'class': MD5, 'size': 12 }, + } + + _key_info = { + 'ssh-rsa': RSAKey, + 'ssh-dss': DSSKey, + 'ecdsa-sha2-nistp256': ECDSAKey, + } + + _kex_info = { + 'diffie-hellman-group1-sha1': KexGroup1, + 'diffie-hellman-group-exchange-sha1': KexGex, + } + + _compression_info = { + # zlib@openssh.com is just zlib, but only turned on after a successful + # authentication. openssh servers may only offer this type because + # they've had troubles with security holes in zlib in the past. + 'zlib@openssh.com': ( ZlibCompressor, ZlibDecompressor ), + 'zlib': ( ZlibCompressor, ZlibDecompressor ), + 'none': ( None, None ), + } + + + _modulus_pack = None + + def __init__(self, sock): + """ + Create a new SSH session over an existing socket, or socket-like + object. This only creates the Transport object; it doesn't begin the + SSH session yet. Use L{connect} or L{start_client} to begin a client + session, or L{start_server} to begin a server session. + + If the object is not actually a socket, it must have the following + methods: + - C{send(str)}: Writes from 1 to C{len(str)} bytes, and + returns an int representing the number of bytes written. Returns + 0 or raises C{EOFError} if the stream has been closed. + - C{recv(int)}: Reads from 1 to C{int} bytes and returns them as a + string. Returns 0 or raises C{EOFError} if the stream has been + closed. + - C{close()}: Closes the socket. + - C{settimeout(n)}: Sets a (float) timeout on I/O operations. + + For ease of use, you may also pass in an address (as a tuple) or a host + string as the C{sock} argument. (A host string is a hostname with an + optional port (separated by C{":"}) which will be converted into a + tuple of C{(hostname, port)}.) A socket will be connected to this + address and used for communication. Exceptions from the C{socket} call + may be thrown in this case. + + @param sock: a socket or socket-like object to create the session over. + @type sock: socket + """ + if isinstance(sock, (str, unicode)): + # convert "host:port" into (host, port) + hl = sock.split(':', 1) + if len(hl) == 1: + sock = (hl[0], 22) + else: + sock = (hl[0], int(hl[1])) + if type(sock) is tuple: + # connect to the given (host, port) + hostname, port = sock + reason = 'No suitable address family' + for (family, socktype, proto, canonname, sockaddr) in socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM): + if socktype == socket.SOCK_STREAM: + af = family + addr = sockaddr + sock = socket.socket(af, socket.SOCK_STREAM) + try: + retry_on_signal(lambda: sock.connect((hostname, port))) + except socket.error, e: + reason = str(e) + else: + break + else: + raise SSHException( + 'Unable to connect to %s: %s' % (hostname, reason)) + # okay, normal socket-ish flow here... + threading.Thread.__init__(self) + self.setDaemon(True) + self.rng = rng + self.sock = sock + # Python < 2.3 doesn't have the settimeout method - RogerB + try: + # we set the timeout so we can check self.active periodically to + # see if we should bail. socket.timeout exception is never + # propagated. + self.sock.settimeout(0.1) + except AttributeError: + pass + + # negotiated crypto parameters + self.packetizer = Packetizer(sock) + self.local_version = 'SSH-' + self._PROTO_ID + '-' + self._CLIENT_ID + self.remote_version = '' + self.local_cipher = self.remote_cipher = '' + self.local_kex_init = self.remote_kex_init = None + self.local_mac = self.remote_mac = None + self.local_compression = self.remote_compression = None + self.session_id = None + self.host_key_type = None + self.host_key = None + + # state used during negotiation + self.kex_engine = None + self.H = None + self.K = None + + self.active = False + self.initial_kex_done = False + self.in_kex = False + self.authenticated = False + self._expected_packet = tuple() + self.lock = threading.Lock() # synchronization (always higher level than write_lock) + + # tracking open channels + self._channels = ChannelMap() + self.channel_events = { } # (id -> Event) + self.channels_seen = { } # (id -> True) + self._channel_counter = 1 + self.window_size = 65536 + self.max_packet_size = 34816 + self._forward_agent_handler = None + self._x11_handler = None + self._tcp_handler = None + + self.saved_exception = None + self.clear_to_send = threading.Event() + self.clear_to_send_lock = threading.Lock() + self.clear_to_send_timeout = 30.0 + self.log_name = 'paramiko.transport' + self.logger = util.get_logger(self.log_name) + self.packetizer.set_log(self.logger) + self.auth_handler = None + self.global_response = None # response Message from an arbitrary global request + self.completion_event = None # user-defined event callbacks + self.banner_timeout = 15 # how long (seconds) to wait for the SSH banner + + # server mode: + self.server_mode = False + self.server_object = None + self.server_key_dict = { } + self.server_accepts = [ ] + self.server_accept_cv = threading.Condition(self.lock) + self.subsystem_table = { } + + def __repr__(self): + """ + Returns a string representation of this object, for debugging. + + @rtype: str + """ + out = '} or + L{auth_publickey }. + + @note: L{connect} is a simpler method for connecting as a client. + + @note: After calling this method (or L{start_server} or L{connect}), + you should no longer directly read from or write to the original + socket object. + + @param event: an event to trigger when negotiation is complete + (optional) + @type event: threading.Event + + @raise SSHException: if negotiation fails (and no C{event} was passed + in) + """ + self.active = True + if event is not None: + # async, return immediately and let the app poll for completion + self.completion_event = event + self.start() + return + + # synchronous, wait for a result + self.completion_event = event = threading.Event() + self.start() + Random.atfork() + while True: + event.wait(0.1) + if not self.active: + e = self.get_exception() + if e is not None: + raise e + raise SSHException('Negotiation failed.') + if event.isSet(): + break + + def start_server(self, event=None, server=None): + """ + Negotiate a new SSH2 session as a server. This is the first step after + creating a new L{Transport} and setting up your server host key(s). A + separate thread is created for protocol negotiation. + + If an event is passed in, this method returns immediately. When + negotiation is done (successful or not), the given C{Event} will + be triggered. On failure, L{is_active} will return C{False}. + + (Since 1.4) If C{event} is C{None}, this method will not return until + negotation is done. On success, the method returns normally. + Otherwise an SSHException is raised. + + After a successful negotiation, the client will need to authenticate. + Override the methods + L{get_allowed_auths }, + L{check_auth_none }, + L{check_auth_password }, and + L{check_auth_publickey } in the + given C{server} object to control the authentication process. + + After a successful authentication, the client should request to open + a channel. Override + L{check_channel_request } in the + given C{server} object to allow channels to be opened. + + @note: After calling this method (or L{start_client} or L{connect}), + you should no longer directly read from or write to the original + socket object. + + @param event: an event to trigger when negotiation is complete. + @type event: threading.Event + @param server: an object used to perform authentication and create + L{Channel}s. + @type server: L{server.ServerInterface} + + @raise SSHException: if negotiation fails (and no C{event} was passed + in) + """ + if server is None: + server = ServerInterface() + self.server_mode = True + self.server_object = server + self.active = True + if event is not None: + # async, return immediately and let the app poll for completion + self.completion_event = event + self.start() + return + + # synchronous, wait for a result + self.completion_event = event = threading.Event() + self.start() + while True: + event.wait(0.1) + if not self.active: + e = self.get_exception() + if e is not None: + raise e + raise SSHException('Negotiation failed.') + if event.isSet(): + break + + def add_server_key(self, key): + """ + Add a host key to the list of keys used for server mode. When behaving + as a server, the host key is used to sign certain packets during the + SSH2 negotiation, so that the client can trust that we are who we say + we are. Because this is used for signing, the key must contain private + key info, not just the public half. Only one key of each type (RSA or + DSS) is kept. + + @param key: the host key to add, usually an L{RSAKey } or + L{DSSKey }. + @type key: L{PKey } + """ + self.server_key_dict[key.get_name()] = key + + def get_server_key(self): + """ + Return the active host key, in server mode. After negotiating with the + client, this method will return the negotiated host key. If only one + type of host key was set with L{add_server_key}, that's the only key + that will ever be returned. But in cases where you have set more than + one type of host key (for example, an RSA key and a DSS key), the key + type will be negotiated by the client, and this method will return the + key of the type agreed on. If the host key has not been negotiated + yet, C{None} is returned. In client mode, the behavior is undefined. + + @return: host key of the type negotiated by the client, or C{None}. + @rtype: L{PKey } + """ + try: + return self.server_key_dict[self.host_key_type] + except KeyError: + pass + return None + + def load_server_moduli(filename=None): + """ + I{(optional)} + Load a file of prime moduli for use in doing group-exchange key + negotiation in server mode. It's a rather obscure option and can be + safely ignored. + + In server mode, the remote client may request "group-exchange" key + negotiation, which asks the server to send a random prime number that + fits certain criteria. These primes are pretty difficult to compute, + so they can't be generated on demand. But many systems contain a file + of suitable primes (usually named something like C{/etc/ssh/moduli}). + If you call C{load_server_moduli} and it returns C{True}, then this + file of primes has been loaded and we will support "group-exchange" in + server mode. Otherwise server mode will just claim that it doesn't + support that method of key negotiation. + + @param filename: optional path to the moduli file, if you happen to + know that it's not in a standard location. + @type filename: str + @return: True if a moduli file was successfully loaded; False + otherwise. + @rtype: bool + + @note: This has no effect when used in client mode. + """ + Transport._modulus_pack = ModulusPack(rng) + # places to look for the openssh "moduli" file + file_list = [ '/etc/ssh/moduli', '/usr/local/etc/moduli' ] + if filename is not None: + file_list.insert(0, filename) + for fn in file_list: + try: + Transport._modulus_pack.read_file(fn) + return True + except IOError: + pass + # none succeeded + Transport._modulus_pack = None + return False + load_server_moduli = staticmethod(load_server_moduli) + + def close(self): + """ + Close this session, and any open channels that are tied to it. + """ + if not self.active: + return + self.stop_thread() + for chan in self._channels.values(): + chan._unlink() + self.sock.close() + + def get_remote_server_key(self): + """ + Return the host key of the server (in client mode). + + @note: Previously this call returned a tuple of (key type, key string). + You can get the same effect by calling + L{PKey.get_name } for the key type, and + C{str(key)} for the key string. + + @raise SSHException: if no session is currently active. + + @return: public key of the remote server + @rtype: L{PKey } + """ + if (not self.active) or (not self.initial_kex_done): + raise SSHException('No existing session') + return self.host_key + + def is_active(self): + """ + Return true if this session is active (open). + + @return: True if the session is still active (open); False if the + session is closed + @rtype: bool + """ + return self.active + + def open_session(self): + """ + Request a new channel to the server, of type C{"session"}. This + is just an alias for C{open_channel('session')}. + + @return: a new L{Channel} + @rtype: L{Channel} + + @raise SSHException: if the request is rejected or the session ends + prematurely + """ + return self.open_channel('session') + + def open_x11_channel(self, src_addr=None): + """ + Request a new channel to the client, of type C{"x11"}. This + is just an alias for C{open_channel('x11', src_addr=src_addr)}. + + @param src_addr: the source address of the x11 server (port is the + x11 port, ie. 6010) + @type src_addr: (str, int) + @return: a new L{Channel} + @rtype: L{Channel} + + @raise SSHException: if the request is rejected or the session ends + prematurely + """ + return self.open_channel('x11', src_addr=src_addr) + + def open_forward_agent_channel(self): + """ + Request a new channel to the client, of type + C{"auth-agent@openssh.com"}. + + This is just an alias for C{open_channel('auth-agent@openssh.com')}. + @return: a new L{Channel} + @rtype: L{Channel} + + @raise SSHException: if the request is rejected or the session ends + prematurely + """ + return self.open_channel('auth-agent@openssh.com') + + def open_forwarded_tcpip_channel(self, (src_addr, src_port), (dest_addr, dest_port)): + """ + Request a new channel back to the client, of type C{"forwarded-tcpip"}. + This is used after a client has requested port forwarding, for sending + incoming connections back to the client. + + @param src_addr: originator's address + @param src_port: originator's port + @param dest_addr: local (server) connected address + @param dest_port: local (server) connected port + """ + return self.open_channel('forwarded-tcpip', (dest_addr, dest_port), (src_addr, src_port)) + + def open_channel(self, kind, dest_addr=None, src_addr=None): + """ + Request a new channel to the server. L{Channel}s are socket-like + objects used for the actual transfer of data across the session. + You may only request a channel after negotiating encryption (using + L{connect} or L{start_client}) and authenticating. + + @param kind: the kind of channel requested (usually C{"session"}, + C{"forwarded-tcpip"}, C{"direct-tcpip"}, or C{"x11"}) + @type kind: str + @param dest_addr: the destination address of this port forwarding, + if C{kind} is C{"forwarded-tcpip"} or C{"direct-tcpip"} (ignored + for other channel types) + @type dest_addr: (str, int) + @param src_addr: the source address of this port forwarding, if + C{kind} is C{"forwarded-tcpip"}, C{"direct-tcpip"}, or C{"x11"} + @type src_addr: (str, int) + @return: a new L{Channel} on success + @rtype: L{Channel} + + @raise SSHException: if the request is rejected or the session ends + prematurely + """ + if not self.active: + raise SSHException('SSH session not active') + self.lock.acquire() + try: + chanid = self._next_channel() + m = Message() + m.add_byte(chr(MSG_CHANNEL_OPEN)) + m.add_string(kind) + m.add_int(chanid) + m.add_int(self.window_size) + m.add_int(self.max_packet_size) + if (kind == 'forwarded-tcpip') or (kind == 'direct-tcpip'): + m.add_string(dest_addr[0]) + m.add_int(dest_addr[1]) + m.add_string(src_addr[0]) + m.add_int(src_addr[1]) + elif kind == 'x11': + m.add_string(src_addr[0]) + m.add_int(src_addr[1]) + chan = Channel(chanid) + self._channels.put(chanid, chan) + self.channel_events[chanid] = event = threading.Event() + self.channels_seen[chanid] = True + chan._set_transport(self) + chan._set_window(self.window_size, self.max_packet_size) + finally: + self.lock.release() + self._send_user_message(m) + while True: + event.wait(0.1); + if not self.active: + e = self.get_exception() + if e is None: + e = SSHException('Unable to open channel.') + raise e + if event.isSet(): + break + chan = self._channels.get(chanid) + if chan is not None: + return chan + e = self.get_exception() + if e is None: + e = SSHException('Unable to open channel.') + raise e + + def request_port_forward(self, address, port, handler=None): + """ + Ask the server to forward TCP connections from a listening port on + the server, across this SSH session. + + If a handler is given, that handler is called from a different thread + whenever a forwarded connection arrives. The handler parameters are:: + + handler(channel, (origin_addr, origin_port), (server_addr, server_port)) + + where C{server_addr} and C{server_port} are the address and port that + the server was listening on. + + If no handler is set, the default behavior is to send new incoming + forwarded connections into the accept queue, to be picked up via + L{accept}. + + @param address: the address to bind when forwarding + @type address: str + @param port: the port to forward, or 0 to ask the server to allocate + any port + @type port: int + @param handler: optional handler for incoming forwarded connections + @type handler: function(Channel, (str, int), (str, int)) + @return: the port # allocated by the server + @rtype: int + + @raise SSHException: if the server refused the TCP forward request + """ + if not self.active: + raise SSHException('SSH session not active') + address = str(address) + port = int(port) + response = self.global_request('tcpip-forward', (address, port), wait=True) + if response is None: + raise SSHException('TCP forwarding request denied') + if port == 0: + port = response.get_int() + if handler is None: + def default_handler(channel, (src_addr, src_port), (dest_addr, dest_port)): + self._queue_incoming_channel(channel) + handler = default_handler + self._tcp_handler = handler + return port + + def cancel_port_forward(self, address, port): + """ + Ask the server to cancel a previous port-forwarding request. No more + connections to the given address & port will be forwarded across this + ssh connection. + + @param address: the address to stop forwarding + @type address: str + @param port: the port to stop forwarding + @type port: int + """ + if not self.active: + return + self._tcp_handler = None + self.global_request('cancel-tcpip-forward', (address, port), wait=True) + + def open_sftp_client(self): + """ + Create an SFTP client channel from an open transport. On success, + an SFTP session will be opened with the remote host, and a new + SFTPClient object will be returned. + + @return: a new L{SFTPClient} object, referring to an sftp session + (channel) across this transport + @rtype: L{SFTPClient} + """ + return SFTPClient.from_transport(self) + + def send_ignore(self, bytes=None): + """ + Send a junk packet across the encrypted link. This is sometimes used + to add "noise" to a connection to confuse would-be attackers. It can + also be used as a keep-alive for long lived connections traversing + firewalls. + + @param bytes: the number of random bytes to send in the payload of the + ignored packet -- defaults to a random number from 10 to 41. + @type bytes: int + """ + m = Message() + m.add_byte(chr(MSG_IGNORE)) + if bytes is None: + bytes = (ord(rng.read(1)) % 32) + 10 + m.add_bytes(rng.read(bytes)) + self._send_user_message(m) + + def renegotiate_keys(self): + """ + Force this session to switch to new keys. Normally this is done + automatically after the session hits a certain number of packets or + bytes sent or received, but this method gives you the option of forcing + new keys whenever you want. Negotiating new keys causes a pause in + traffic both ways as the two sides swap keys and do computations. This + method returns when the session has switched to new keys. + + @raise SSHException: if the key renegotiation failed (which causes the + session to end) + """ + self.completion_event = threading.Event() + self._send_kex_init() + while True: + self.completion_event.wait(0.1) + if not self.active: + e = self.get_exception() + if e is not None: + raise e + raise SSHException('Negotiation failed.') + if self.completion_event.isSet(): + break + return + + def set_keepalive(self, interval): + """ + Turn on/off keepalive packets (default is off). If this is set, after + C{interval} seconds without sending any data over the connection, a + "keepalive" packet will be sent (and ignored by the remote host). This + can be useful to keep connections alive over a NAT, for example. + + @param interval: seconds to wait before sending a keepalive packet (or + 0 to disable keepalives). + @type interval: int + """ + self.packetizer.set_keepalive(interval, + lambda x=weakref.proxy(self): x.global_request('keepalive@lag.net', wait=False)) + + def global_request(self, kind, data=None, wait=True): + """ + Make a global request to the remote host. These are normally + extensions to the SSH2 protocol. + + @param kind: name of the request. + @type kind: str + @param data: an optional tuple containing additional data to attach + to the request. + @type data: tuple + @param wait: C{True} if this method should not return until a response + is received; C{False} otherwise. + @type wait: bool + @return: a L{Message} containing possible additional data if the + request was successful (or an empty L{Message} if C{wait} was + C{False}); C{None} if the request was denied. + @rtype: L{Message} + """ + if wait: + self.completion_event = threading.Event() + m = Message() + m.add_byte(chr(MSG_GLOBAL_REQUEST)) + m.add_string(kind) + m.add_boolean(wait) + if data is not None: + m.add(*data) + self._log(DEBUG, 'Sending global request "%s"' % kind) + self._send_user_message(m) + if not wait: + return None + while True: + self.completion_event.wait(0.1) + if not self.active: + return None + if self.completion_event.isSet(): + break + return self.global_response + + def accept(self, timeout=None): + """ + Return the next channel opened by the client over this transport, in + server mode. If no channel is opened before the given timeout, C{None} + is returned. + + @param timeout: seconds to wait for a channel, or C{None} to wait + forever + @type timeout: int + @return: a new Channel opened by the client + @rtype: L{Channel} + """ + self.lock.acquire() + try: + if len(self.server_accepts) > 0: + chan = self.server_accepts.pop(0) + else: + self.server_accept_cv.wait(timeout) + if len(self.server_accepts) > 0: + chan = self.server_accepts.pop(0) + else: + # timeout + chan = None + finally: + self.lock.release() + return chan + + def connect(self, hostkey=None, username='', password=None, pkey=None): + """ + Negotiate an SSH2 session, and optionally verify the server's host key + and authenticate using a password or private key. This is a shortcut + for L{start_client}, L{get_remote_server_key}, and + L{Transport.auth_password} or L{Transport.auth_publickey}. Use those + methods if you want more control. + + You can use this method immediately after creating a Transport to + negotiate encryption with a server. If it fails, an exception will be + thrown. On success, the method will return cleanly, and an encrypted + session exists. You may immediately call L{open_channel} or + L{open_session} to get a L{Channel} object, which is used for data + transfer. + + @note: If you fail to supply a password or private key, this method may + succeed, but a subsequent L{open_channel} or L{open_session} call may + fail because you haven't authenticated yet. + + @param hostkey: the host key expected from the server, or C{None} if + you don't want to do host key verification. + @type hostkey: L{PKey} + @param username: the username to authenticate as. + @type username: str + @param password: a password to use for authentication, if you want to + use password authentication; otherwise C{None}. + @type password: str + @param pkey: a private key to use for authentication, if you want to + use private key authentication; otherwise C{None}. + @type pkey: L{PKey} + + @raise SSHException: if the SSH2 negotiation fails, the host key + supplied by the server is incorrect, or authentication fails. + """ + if hostkey is not None: + self._preferred_keys = [ hostkey.get_name() ] + + self.start_client() + + # check host key if we were given one + if (hostkey is not None): + key = self.get_remote_server_key() + if (key.get_name() != hostkey.get_name()) or (str(key) != str(hostkey)): + self._log(DEBUG, 'Bad host key from server') + self._log(DEBUG, 'Expected: %s: %s' % (hostkey.get_name(), repr(str(hostkey)))) + self._log(DEBUG, 'Got : %s: %s' % (key.get_name(), repr(str(key)))) + raise SSHException('Bad host key from server') + self._log(DEBUG, 'Host key verified (%s)' % hostkey.get_name()) + + if (pkey is not None) or (password is not None): + if password is not None: + self._log(DEBUG, 'Attempting password auth...') + self.auth_password(username, password) + else: + self._log(DEBUG, 'Attempting public-key auth...') + self.auth_publickey(username, pkey) + + return + + def get_exception(self): + """ + Return any exception that happened during the last server request. + This can be used to fetch more specific error information after using + calls like L{start_client}. The exception (if any) is cleared after + this call. + + @return: an exception, or C{None} if there is no stored exception. + @rtype: Exception + + @since: 1.1 + """ + self.lock.acquire() + try: + e = self.saved_exception + self.saved_exception = None + return e + finally: + self.lock.release() + + def set_subsystem_handler(self, name, handler, *larg, **kwarg): + """ + Set the handler class for a subsystem in server mode. If a request + for this subsystem is made on an open ssh channel later, this handler + will be constructed and called -- see L{SubsystemHandler} for more + detailed documentation. + + Any extra parameters (including keyword arguments) are saved and + passed to the L{SubsystemHandler} constructor later. + + @param name: name of the subsystem. + @type name: str + @param handler: subclass of L{SubsystemHandler} that handles this + subsystem. + @type handler: class + """ + try: + self.lock.acquire() + self.subsystem_table[name] = (handler, larg, kwarg) + finally: + self.lock.release() + + def is_authenticated(self): + """ + Return true if this session is active and authenticated. + + @return: True if the session is still open and has been authenticated + successfully; False if authentication failed and/or the session is + closed. + @rtype: bool + """ + return self.active and (self.auth_handler is not None) and self.auth_handler.is_authenticated() + + def get_username(self): + """ + Return the username this connection is authenticated for. If the + session is not authenticated (or authentication failed), this method + returns C{None}. + + @return: username that was authenticated, or C{None}. + @rtype: string + """ + if not self.active or (self.auth_handler is None): + return None + return self.auth_handler.get_username() + + def auth_none(self, username): + """ + Try to authenticate to the server using no authentication at all. + This will almost always fail. It may be useful for determining the + list of authentication types supported by the server, by catching the + L{BadAuthenticationType} exception raised. + + @param username: the username to authenticate as + @type username: string + @return: list of auth types permissible for the next stage of + authentication (normally empty) + @rtype: list + + @raise BadAuthenticationType: if "none" authentication isn't allowed + by the server for this user + @raise SSHException: if the authentication failed due to a network + error + + @since: 1.5 + """ + if (not self.active) or (not self.initial_kex_done): + raise SSHException('No existing session') + my_event = threading.Event() + self.auth_handler = AuthHandler(self) + self.auth_handler.auth_none(username, my_event) + return self.auth_handler.wait_for_response(my_event) + + def auth_password(self, username, password, event=None, fallback=True): + """ + Authenticate to the server using a password. The username and password + are sent over an encrypted link. + + If an C{event} is passed in, this method will return immediately, and + the event will be triggered once authentication succeeds or fails. On + success, L{is_authenticated} will return C{True}. On failure, you may + use L{get_exception} to get more detailed error information. + + Since 1.1, if no event is passed, this method will block until the + authentication succeeds or fails. On failure, an exception is raised. + Otherwise, the method simply returns. + + Since 1.5, if no event is passed and C{fallback} is C{True} (the + default), if the server doesn't support plain password authentication + but does support so-called "keyboard-interactive" mode, an attempt + will be made to authenticate using this interactive mode. If it fails, + the normal exception will be thrown as if the attempt had never been + made. This is useful for some recent Gentoo and Debian distributions, + which turn off plain password authentication in a misguided belief + that interactive authentication is "more secure". (It's not.) + + If the server requires multi-step authentication (which is very rare), + this method will return a list of auth types permissible for the next + step. Otherwise, in the normal case, an empty list is returned. + + @param username: the username to authenticate as + @type username: str + @param password: the password to authenticate with + @type password: str or unicode + @param event: an event to trigger when the authentication attempt is + complete (whether it was successful or not) + @type event: threading.Event + @param fallback: C{True} if an attempt at an automated "interactive" + password auth should be made if the server doesn't support normal + password auth + @type fallback: bool + @return: list of auth types permissible for the next stage of + authentication (normally empty) + @rtype: list + + @raise BadAuthenticationType: if password authentication isn't + allowed by the server for this user (and no event was passed in) + @raise AuthenticationException: if the authentication failed (and no + event was passed in) + @raise SSHException: if there was a network error + """ + if (not self.active) or (not self.initial_kex_done): + # we should never try to send the password unless we're on a secure link + raise SSHException('No existing session') + if event is None: + my_event = threading.Event() + else: + my_event = event + self.auth_handler = AuthHandler(self) + self.auth_handler.auth_password(username, password, my_event) + if event is not None: + # caller wants to wait for event themselves + return [] + try: + return self.auth_handler.wait_for_response(my_event) + except BadAuthenticationType, x: + # if password auth isn't allowed, but keyboard-interactive *is*, try to fudge it + if not fallback or ('keyboard-interactive' not in x.allowed_types): + raise + try: + def handler(title, instructions, fields): + if len(fields) > 1: + raise SSHException('Fallback authentication failed.') + if len(fields) == 0: + # for some reason, at least on os x, a 2nd request will + # be made with zero fields requested. maybe it's just + # to try to fake out automated scripting of the exact + # type we're doing here. *shrug* :) + return [] + return [ password ] + return self.auth_interactive(username, handler) + except SSHException, ignored: + # attempt failed; just raise the original exception + raise x + return None + + def auth_publickey(self, username, key, event=None): + """ + Authenticate to the server using a private key. The key is used to + sign data from the server, so it must include the private part. + + If an C{event} is passed in, this method will return immediately, and + the event will be triggered once authentication succeeds or fails. On + success, L{is_authenticated} will return C{True}. On failure, you may + use L{get_exception} to get more detailed error information. + + Since 1.1, if no event is passed, this method will block until the + authentication succeeds or fails. On failure, an exception is raised. + Otherwise, the method simply returns. + + If the server requires multi-step authentication (which is very rare), + this method will return a list of auth types permissible for the next + step. Otherwise, in the normal case, an empty list is returned. + + @param username: the username to authenticate as + @type username: string + @param key: the private key to authenticate with + @type key: L{PKey } + @param event: an event to trigger when the authentication attempt is + complete (whether it was successful or not) + @type event: threading.Event + @return: list of auth types permissible for the next stage of + authentication (normally empty) + @rtype: list + + @raise BadAuthenticationType: if public-key authentication isn't + allowed by the server for this user (and no event was passed in) + @raise AuthenticationException: if the authentication failed (and no + event was passed in) + @raise SSHException: if there was a network error + """ + if (not self.active) or (not self.initial_kex_done): + # we should never try to authenticate unless we're on a secure link + raise SSHException('No existing session') + if event is None: + my_event = threading.Event() + else: + my_event = event + self.auth_handler = AuthHandler(self) + self.auth_handler.auth_publickey(username, key, my_event) + if event is not None: + # caller wants to wait for event themselves + return [] + return self.auth_handler.wait_for_response(my_event) + + def auth_interactive(self, username, handler, submethods=''): + """ + Authenticate to the server interactively. A handler is used to answer + arbitrary questions from the server. On many servers, this is just a + dumb wrapper around PAM. + + This method will block until the authentication succeeds or fails, + peroidically calling the handler asynchronously to get answers to + authentication questions. The handler may be called more than once + if the server continues to ask questions. + + The handler is expected to be a callable that will handle calls of the + form: C{handler(title, instructions, prompt_list)}. The C{title} is + meant to be a dialog-window title, and the C{instructions} are user + instructions (both are strings). C{prompt_list} will be a list of + prompts, each prompt being a tuple of C{(str, bool)}. The string is + the prompt and the boolean indicates whether the user text should be + echoed. + + A sample call would thus be: + C{handler('title', 'instructions', [('Password:', False)])}. + + The handler should return a list or tuple of answers to the server's + questions. + + If the server requires multi-step authentication (which is very rare), + this method will return a list of auth types permissible for the next + step. Otherwise, in the normal case, an empty list is returned. + + @param username: the username to authenticate as + @type username: string + @param handler: a handler for responding to server questions + @type handler: callable + @param submethods: a string list of desired submethods (optional) + @type submethods: str + @return: list of auth types permissible for the next stage of + authentication (normally empty). + @rtype: list + + @raise BadAuthenticationType: if public-key authentication isn't + allowed by the server for this user + @raise AuthenticationException: if the authentication failed + @raise SSHException: if there was a network error + + @since: 1.5 + """ + if (not self.active) or (not self.initial_kex_done): + # we should never try to authenticate unless we're on a secure link + raise SSHException('No existing session') + my_event = threading.Event() + self.auth_handler = AuthHandler(self) + self.auth_handler.auth_interactive(username, handler, my_event, submethods) + return self.auth_handler.wait_for_response(my_event) + + def set_log_channel(self, name): + """ + Set the channel for this transport's logging. The default is + C{"paramiko.transport"} but it can be set to anything you want. + (See the C{logging} module for more info.) SSH Channels will log + to a sub-channel of the one specified. + + @param name: new channel name for logging + @type name: str + + @since: 1.1 + """ + self.log_name = name + self.logger = util.get_logger(name) + self.packetizer.set_log(self.logger) + + def get_log_channel(self): + """ + Return the channel name used for this transport's logging. + + @return: channel name. + @rtype: str + + @since: 1.2 + """ + return self.log_name + + def set_hexdump(self, hexdump): + """ + Turn on/off logging a hex dump of protocol traffic at DEBUG level in + the logs. Normally you would want this off (which is the default), + but if you are debugging something, it may be useful. + + @param hexdump: C{True} to log protocol traffix (in hex) to the log; + C{False} otherwise. + @type hexdump: bool + """ + self.packetizer.set_hexdump(hexdump) + + def get_hexdump(self): + """ + Return C{True} if the transport is currently logging hex dumps of + protocol traffic. + + @return: C{True} if hex dumps are being logged + @rtype: bool + + @since: 1.4 + """ + return self.packetizer.get_hexdump() + + def use_compression(self, compress=True): + """ + Turn on/off compression. This will only have an affect before starting + the transport (ie before calling L{connect}, etc). By default, + compression is off since it negatively affects interactive sessions. + + @param compress: C{True} to ask the remote client/server to compress + traffic; C{False} to refuse compression + @type compress: bool + + @since: 1.5.2 + """ + if compress: + self._preferred_compression = ( 'zlib@openssh.com', 'zlib', 'none' ) + else: + self._preferred_compression = ( 'none', ) + + def getpeername(self): + """ + Return the address of the remote side of this Transport, if possible. + This is effectively a wrapper around C{'getpeername'} on the underlying + socket. If the socket-like object has no C{'getpeername'} method, + then C{("unknown", 0)} is returned. + + @return: the address if the remote host, if known + @rtype: tuple(str, int) + """ + gp = getattr(self.sock, 'getpeername', None) + if gp is None: + return ('unknown', 0) + return gp() + + def stop_thread(self): + self.active = False + self.packetizer.close() + while self.isAlive(): + self.join(10) + + + ### internals... + + + def _log(self, level, msg, *args): + if issubclass(type(msg), list): + for m in msg: + self.logger.log(level, m) + else: + self.logger.log(level, msg, *args) + + def _get_modulus_pack(self): + "used by KexGex to find primes for group exchange" + return self._modulus_pack + + def _next_channel(self): + "you are holding the lock" + chanid = self._channel_counter + while self._channels.get(chanid) is not None: + self._channel_counter = (self._channel_counter + 1) & 0xffffff + chanid = self._channel_counter + self._channel_counter = (self._channel_counter + 1) & 0xffffff + return chanid + + def _unlink_channel(self, chanid): + "used by a Channel to remove itself from the active channel list" + self._channels.delete(chanid) + + def _send_message(self, data): + self.packetizer.send_message(data) + + def _send_user_message(self, data): + """ + send a message, but block if we're in key negotiation. this is used + for user-initiated requests. + """ + start = time.time() + while True: + self.clear_to_send.wait(0.1) + if not self.active: + self._log(DEBUG, 'Dropping user packet because connection is dead.') + return + self.clear_to_send_lock.acquire() + if self.clear_to_send.isSet(): + break + self.clear_to_send_lock.release() + if time.time() > start + self.clear_to_send_timeout: + raise SSHException('Key-exchange timed out waiting for key negotiation') + try: + self._send_message(data) + finally: + self.clear_to_send_lock.release() + + def _set_K_H(self, k, h): + "used by a kex object to set the K (root key) and H (exchange hash)" + self.K = k + self.H = h + if self.session_id == None: + self.session_id = h + + def _expect_packet(self, *ptypes): + "used by a kex object to register the next packet type it expects to see" + self._expected_packet = tuple(ptypes) + + def _verify_key(self, host_key, sig): + key = self._key_info[self.host_key_type](Message(host_key)) + if key is None: + raise SSHException('Unknown host key type') + if not key.verify_ssh_sig(self.H, Message(sig)): + raise SSHException('Signature verification (%s) failed.' % self.host_key_type) + self.host_key = key + + def _compute_key(self, id, nbytes): + "id is 'A' - 'F' for the various keys used by ssh" + m = Message() + m.add_mpint(self.K) + m.add_bytes(self.H) + m.add_byte(id) + m.add_bytes(self.session_id) + out = sofar = SHA.new(str(m)).digest() + while len(out) < nbytes: + m = Message() + m.add_mpint(self.K) + m.add_bytes(self.H) + m.add_bytes(sofar) + digest = SHA.new(str(m)).digest() + out += digest + sofar += digest + return out[:nbytes] + + def _get_cipher(self, name, key, iv): + if name not in self._cipher_info: + raise SSHException('Unknown client cipher ' + name) + if name in ('arcfour128', 'arcfour256'): + # arcfour cipher + cipher = self._cipher_info[name]['class'].new(key) + # as per RFC 4345, the first 1536 bytes of keystream + # generated by the cipher MUST be discarded + cipher.encrypt(" " * 1536) + return cipher + elif name.endswith("-ctr"): + # CTR modes, we need a counter + counter = Counter.new(nbits=self._cipher_info[name]['block-size'] * 8, initial_value=util.inflate_long(iv, True)) + return self._cipher_info[name]['class'].new(key, self._cipher_info[name]['mode'], iv, counter) + else: + return self._cipher_info[name]['class'].new(key, self._cipher_info[name]['mode'], iv) + + def _set_forward_agent_handler(self, handler): + if handler is None: + def default_handler(channel): + self._queue_incoming_channel(channel) + self._forward_agent_handler = default_handler + else: + self._forward_agent_handler = handler + + def _set_x11_handler(self, handler): + # only called if a channel has turned on x11 forwarding + if handler is None: + # by default, use the same mechanism as accept() + def default_handler(channel, (src_addr, src_port)): + self._queue_incoming_channel(channel) + self._x11_handler = default_handler + else: + self._x11_handler = handler + + def _queue_incoming_channel(self, channel): + self.lock.acquire() + try: + self.server_accepts.append(channel) + self.server_accept_cv.notify() + finally: + self.lock.release() + + def run(self): + # (use the exposed "run" method, because if we specify a thread target + # of a private method, threading.Thread will keep a reference to it + # indefinitely, creating a GC cycle and not letting Transport ever be + # GC'd. it's a bug in Thread.) + + # Hold reference to 'sys' so we can test sys.modules to detect + # interpreter shutdown. + self.sys = sys + + # Required to prevent RNG errors when running inside many subprocess + # containers. + Random.atfork() + + # Hold reference to 'sys' so we can test sys.modules to detect + # interpreter shutdown. + self.sys = sys + + # active=True occurs before the thread is launched, to avoid a race + _active_threads.append(self) + if self.server_mode: + self._log(DEBUG, 'starting thread (server mode): %s' % hex(long(id(self)) & 0xffffffffL)) + else: + self._log(DEBUG, 'starting thread (client mode): %s' % hex(long(id(self)) & 0xffffffffL)) + try: + try: + self.packetizer.write_all(self.local_version + '\r\n') + self._check_banner() + self._send_kex_init() + self._expect_packet(MSG_KEXINIT) + + while self.active: + if self.packetizer.need_rekey() and not self.in_kex: + self._send_kex_init() + try: + ptype, m = self.packetizer.read_message() + except NeedRekeyException: + continue + if ptype == MSG_IGNORE: + continue + elif ptype == MSG_DISCONNECT: + self._parse_disconnect(m) + self.active = False + self.packetizer.close() + break + elif ptype == MSG_DEBUG: + self._parse_debug(m) + continue + if len(self._expected_packet) > 0: + if ptype not in self._expected_packet: + raise SSHException('Expecting packet from %r, got %d' % (self._expected_packet, ptype)) + self._expected_packet = tuple() + if (ptype >= 30) and (ptype <= 39): + self.kex_engine.parse_next(ptype, m) + continue + + if ptype in self._handler_table: + self._handler_table[ptype](self, m) + elif ptype in self._channel_handler_table: + chanid = m.get_int() + chan = self._channels.get(chanid) + if chan is not None: + self._channel_handler_table[ptype](chan, m) + elif chanid in self.channels_seen: + self._log(DEBUG, 'Ignoring message for dead channel %d' % chanid) + else: + self._log(ERROR, 'Channel request for unknown channel %d' % chanid) + self.active = False + self.packetizer.close() + elif (self.auth_handler is not None) and (ptype in self.auth_handler._handler_table): + self.auth_handler._handler_table[ptype](self.auth_handler, m) + else: + self._log(WARNING, 'Oops, unhandled type %d' % ptype) + msg = Message() + msg.add_byte(chr(MSG_UNIMPLEMENTED)) + msg.add_int(m.seqno) + self._send_message(msg) + except SSHException, e: + self._log(ERROR, 'Exception: ' + str(e)) + self._log(ERROR, util.tb_strings()) + self.saved_exception = e + except EOFError, e: + self._log(DEBUG, 'EOF in transport thread') + #self._log(DEBUG, util.tb_strings()) + self.saved_exception = e + except socket.error, e: + if type(e.args) is tuple: + emsg = '%s (%d)' % (e.args[1], e.args[0]) + else: + emsg = e.args + self._log(ERROR, 'Socket exception: ' + emsg) + self.saved_exception = e + except Exception, e: + self._log(ERROR, 'Unknown exception: ' + str(e)) + self._log(ERROR, util.tb_strings()) + self.saved_exception = e + _active_threads.remove(self) + for chan in self._channels.values(): + chan._unlink() + if self.active: + self.active = False + self.packetizer.close() + if self.completion_event != None: + self.completion_event.set() + if self.auth_handler is not None: + self.auth_handler.abort() + for event in self.channel_events.values(): + event.set() + try: + self.lock.acquire() + self.server_accept_cv.notify() + finally: + self.lock.release() + self.sock.close() + except: + # Don't raise spurious 'NoneType has no attribute X' errors when we + # wake up during interpreter shutdown. Or rather -- raise + # everything *if* sys.modules (used as a convenient sentinel) + # appears to still exist. + if self.sys.modules is not None: + raise + + + ### protocol stages + + + def _negotiate_keys(self, m): + # throws SSHException on anything unusual + self.clear_to_send_lock.acquire() + try: + self.clear_to_send.clear() + finally: + self.clear_to_send_lock.release() + if self.local_kex_init == None: + # remote side wants to renegotiate + self._send_kex_init() + self._parse_kex_init(m) + self.kex_engine.start_kex() + + def _check_banner(self): + # this is slow, but we only have to do it once + for i in range(100): + # give them 15 seconds for the first line, then just 2 seconds + # each additional line. (some sites have very high latency.) + if i == 0: + timeout = self.banner_timeout + else: + timeout = 2 + try: + buf = self.packetizer.readline(timeout) + except ProxyCommandFailure: + raise + except Exception, x: + raise SSHException('Error reading SSH protocol banner' + str(x)) + if buf[:4] == 'SSH-': + break + self._log(DEBUG, 'Banner: ' + buf) + if buf[:4] != 'SSH-': + raise SSHException('Indecipherable protocol version "' + buf + '"') + # save this server version string for later + self.remote_version = buf + # pull off any attached comment + comment = '' + i = string.find(buf, ' ') + if i >= 0: + comment = buf[i+1:] + buf = buf[:i] + # parse out version string and make sure it matches + segs = buf.split('-', 2) + if len(segs) < 3: + raise SSHException('Invalid SSH banner') + version = segs[1] + client = segs[2] + if version != '1.99' and version != '2.0': + raise SSHException('Incompatible version (%s instead of 2.0)' % (version,)) + self._log(INFO, 'Connected (version %s, client %s)' % (version, client)) + + def _send_kex_init(self): + """ + announce to the other side that we'd like to negotiate keys, and what + kind of key negotiation we support. + """ + self.clear_to_send_lock.acquire() + try: + self.clear_to_send.clear() + finally: + self.clear_to_send_lock.release() + self.in_kex = True + if self.server_mode: + if (self._modulus_pack is None) and ('diffie-hellman-group-exchange-sha1' in self._preferred_kex): + # can't do group-exchange if we don't have a pack of potential primes + pkex = list(self.get_security_options().kex) + pkex.remove('diffie-hellman-group-exchange-sha1') + self.get_security_options().kex = pkex + available_server_keys = filter(self.server_key_dict.keys().__contains__, + self._preferred_keys) + else: + available_server_keys = self._preferred_keys + + m = Message() + m.add_byte(chr(MSG_KEXINIT)) + m.add_bytes(rng.read(16)) + m.add_list(self._preferred_kex) + m.add_list(available_server_keys) + m.add_list(self._preferred_ciphers) + m.add_list(self._preferred_ciphers) + m.add_list(self._preferred_macs) + m.add_list(self._preferred_macs) + m.add_list(self._preferred_compression) + m.add_list(self._preferred_compression) + m.add_string('') + m.add_string('') + m.add_boolean(False) + m.add_int(0) + # save a copy for later (needed to compute a hash) + self.local_kex_init = str(m) + self._send_message(m) + + def _parse_kex_init(self, m): + cookie = m.get_bytes(16) + kex_algo_list = m.get_list() + server_key_algo_list = m.get_list() + client_encrypt_algo_list = m.get_list() + server_encrypt_algo_list = m.get_list() + client_mac_algo_list = m.get_list() + server_mac_algo_list = m.get_list() + client_compress_algo_list = m.get_list() + server_compress_algo_list = m.get_list() + client_lang_list = m.get_list() + server_lang_list = m.get_list() + kex_follows = m.get_boolean() + unused = m.get_int() + + self._log(DEBUG, 'kex algos:' + str(kex_algo_list) + ' server key:' + str(server_key_algo_list) + \ + ' client encrypt:' + str(client_encrypt_algo_list) + \ + ' server encrypt:' + str(server_encrypt_algo_list) + \ + ' client mac:' + str(client_mac_algo_list) + \ + ' server mac:' + str(server_mac_algo_list) + \ + ' client compress:' + str(client_compress_algo_list) + \ + ' server compress:' + str(server_compress_algo_list) + \ + ' client lang:' + str(client_lang_list) + \ + ' server lang:' + str(server_lang_list) + \ + ' kex follows?' + str(kex_follows)) + + # as a server, we pick the first item in the client's list that we support. + # as a client, we pick the first item in our list that the server supports. + if self.server_mode: + agreed_kex = filter(self._preferred_kex.__contains__, kex_algo_list) + else: + agreed_kex = filter(kex_algo_list.__contains__, self._preferred_kex) + if len(agreed_kex) == 0: + raise SSHException('Incompatible ssh peer (no acceptable kex algorithm)') + self.kex_engine = self._kex_info[agreed_kex[0]](self) + + if self.server_mode: + available_server_keys = filter(self.server_key_dict.keys().__contains__, + self._preferred_keys) + agreed_keys = filter(available_server_keys.__contains__, server_key_algo_list) + else: + agreed_keys = filter(server_key_algo_list.__contains__, self._preferred_keys) + if len(agreed_keys) == 0: + raise SSHException('Incompatible ssh peer (no acceptable host key)') + self.host_key_type = agreed_keys[0] + if self.server_mode and (self.get_server_key() is None): + raise SSHException('Incompatible ssh peer (can\'t match requested host key type)') + + if self.server_mode: + agreed_local_ciphers = filter(self._preferred_ciphers.__contains__, + server_encrypt_algo_list) + agreed_remote_ciphers = filter(self._preferred_ciphers.__contains__, + client_encrypt_algo_list) + else: + agreed_local_ciphers = filter(client_encrypt_algo_list.__contains__, + self._preferred_ciphers) + agreed_remote_ciphers = filter(server_encrypt_algo_list.__contains__, + self._preferred_ciphers) + if (len(agreed_local_ciphers) == 0) or (len(agreed_remote_ciphers) == 0): + raise SSHException('Incompatible ssh server (no acceptable ciphers)') + self.local_cipher = agreed_local_ciphers[0] + self.remote_cipher = agreed_remote_ciphers[0] + self._log(DEBUG, 'Ciphers agreed: local=%s, remote=%s' % (self.local_cipher, self.remote_cipher)) + + if self.server_mode: + agreed_remote_macs = filter(self._preferred_macs.__contains__, client_mac_algo_list) + agreed_local_macs = filter(self._preferred_macs.__contains__, server_mac_algo_list) + else: + agreed_local_macs = filter(client_mac_algo_list.__contains__, self._preferred_macs) + agreed_remote_macs = filter(server_mac_algo_list.__contains__, self._preferred_macs) + if (len(agreed_local_macs) == 0) or (len(agreed_remote_macs) == 0): + raise SSHException('Incompatible ssh server (no acceptable macs)') + self.local_mac = agreed_local_macs[0] + self.remote_mac = agreed_remote_macs[0] + + if self.server_mode: + agreed_remote_compression = filter(self._preferred_compression.__contains__, client_compress_algo_list) + agreed_local_compression = filter(self._preferred_compression.__contains__, server_compress_algo_list) + else: + agreed_local_compression = filter(client_compress_algo_list.__contains__, self._preferred_compression) + agreed_remote_compression = filter(server_compress_algo_list.__contains__, self._preferred_compression) + if (len(agreed_local_compression) == 0) or (len(agreed_remote_compression) == 0): + raise SSHException('Incompatible ssh server (no acceptable compression) %r %r %r' % (agreed_local_compression, agreed_remote_compression, self._preferred_compression)) + self.local_compression = agreed_local_compression[0] + self.remote_compression = agreed_remote_compression[0] + + self._log(DEBUG, 'using kex %s; server key type %s; cipher: local %s, remote %s; mac: local %s, remote %s; compression: local %s, remote %s' % + (agreed_kex[0], self.host_key_type, self.local_cipher, self.remote_cipher, self.local_mac, + self.remote_mac, self.local_compression, self.remote_compression)) + + # save for computing hash later... + # now wait! openssh has a bug (and others might too) where there are + # actually some extra bytes (one NUL byte in openssh's case) added to + # the end of the packet but not parsed. turns out we need to throw + # away those bytes because they aren't part of the hash. + self.remote_kex_init = chr(MSG_KEXINIT) + m.get_so_far() + + def _activate_inbound(self): + "switch on newly negotiated encryption parameters for inbound traffic" + block_size = self._cipher_info[self.remote_cipher]['block-size'] + if self.server_mode: + IV_in = self._compute_key('A', block_size) + key_in = self._compute_key('C', self._cipher_info[self.remote_cipher]['key-size']) + else: + IV_in = self._compute_key('B', block_size) + key_in = self._compute_key('D', self._cipher_info[self.remote_cipher]['key-size']) + engine = self._get_cipher(self.remote_cipher, key_in, IV_in) + mac_size = self._mac_info[self.remote_mac]['size'] + mac_engine = self._mac_info[self.remote_mac]['class'] + # initial mac keys are done in the hash's natural size (not the potentially truncated + # transmission size) + if self.server_mode: + mac_key = self._compute_key('E', mac_engine.digest_size) + else: + mac_key = self._compute_key('F', mac_engine.digest_size) + self.packetizer.set_inbound_cipher(engine, block_size, mac_engine, mac_size, mac_key) + compress_in = self._compression_info[self.remote_compression][1] + if (compress_in is not None) and ((self.remote_compression != 'zlib@openssh.com') or self.authenticated): + self._log(DEBUG, 'Switching on inbound compression ...') + self.packetizer.set_inbound_compressor(compress_in()) + + def _activate_outbound(self): + "switch on newly negotiated encryption parameters for outbound traffic" + m = Message() + m.add_byte(chr(MSG_NEWKEYS)) + self._send_message(m) + block_size = self._cipher_info[self.local_cipher]['block-size'] + if self.server_mode: + IV_out = self._compute_key('B', block_size) + key_out = self._compute_key('D', self._cipher_info[self.local_cipher]['key-size']) + else: + IV_out = self._compute_key('A', block_size) + key_out = self._compute_key('C', self._cipher_info[self.local_cipher]['key-size']) + engine = self._get_cipher(self.local_cipher, key_out, IV_out) + mac_size = self._mac_info[self.local_mac]['size'] + mac_engine = self._mac_info[self.local_mac]['class'] + # initial mac keys are done in the hash's natural size (not the potentially truncated + # transmission size) + if self.server_mode: + mac_key = self._compute_key('F', mac_engine.digest_size) + else: + mac_key = self._compute_key('E', mac_engine.digest_size) + sdctr = self.local_cipher.endswith('-ctr') + self.packetizer.set_outbound_cipher(engine, block_size, mac_engine, mac_size, mac_key, sdctr) + compress_out = self._compression_info[self.local_compression][0] + if (compress_out is not None) and ((self.local_compression != 'zlib@openssh.com') or self.authenticated): + self._log(DEBUG, 'Switching on outbound compression ...') + self.packetizer.set_outbound_compressor(compress_out()) + if not self.packetizer.need_rekey(): + self.in_kex = False + # we always expect to receive NEWKEYS now + self._expect_packet(MSG_NEWKEYS) + + def _auth_trigger(self): + self.authenticated = True + # delayed initiation of compression + if self.local_compression == 'zlib@openssh.com': + compress_out = self._compression_info[self.local_compression][0] + self._log(DEBUG, 'Switching on outbound compression ...') + self.packetizer.set_outbound_compressor(compress_out()) + if self.remote_compression == 'zlib@openssh.com': + compress_in = self._compression_info[self.remote_compression][1] + self._log(DEBUG, 'Switching on inbound compression ...') + self.packetizer.set_inbound_compressor(compress_in()) + + def _parse_newkeys(self, m): + self._log(DEBUG, 'Switch to new keys ...') + self._activate_inbound() + # can also free a bunch of stuff here + self.local_kex_init = self.remote_kex_init = None + self.K = None + self.kex_engine = None + if self.server_mode and (self.auth_handler is None): + # create auth handler for server mode + self.auth_handler = AuthHandler(self) + if not self.initial_kex_done: + # this was the first key exchange + self.initial_kex_done = True + # send an event? + if self.completion_event != None: + self.completion_event.set() + # it's now okay to send data again (if this was a re-key) + if not self.packetizer.need_rekey(): + self.in_kex = False + self.clear_to_send_lock.acquire() + try: + self.clear_to_send.set() + finally: + self.clear_to_send_lock.release() + return + + def _parse_disconnect(self, m): + code = m.get_int() + desc = m.get_string() + self._log(INFO, 'Disconnect (code %d): %s' % (code, desc)) + + def _parse_global_request(self, m): + kind = m.get_string() + self._log(DEBUG, 'Received global request "%s"' % kind) + want_reply = m.get_boolean() + if not self.server_mode: + self._log(DEBUG, 'Rejecting "%s" global request from server.' % kind) + ok = False + elif kind == 'tcpip-forward': + address = m.get_string() + port = m.get_int() + ok = self.server_object.check_port_forward_request(address, port) + if ok != False: + ok = (ok,) + elif kind == 'cancel-tcpip-forward': + address = m.get_string() + port = m.get_int() + self.server_object.cancel_port_forward_request(address, port) + ok = True + else: + ok = self.server_object.check_global_request(kind, m) + extra = () + if type(ok) is tuple: + extra = ok + ok = True + if want_reply: + msg = Message() + if ok: + msg.add_byte(chr(MSG_REQUEST_SUCCESS)) + msg.add(*extra) + else: + msg.add_byte(chr(MSG_REQUEST_FAILURE)) + self._send_message(msg) + + def _parse_request_success(self, m): + self._log(DEBUG, 'Global request successful.') + self.global_response = m + if self.completion_event is not None: + self.completion_event.set() + + def _parse_request_failure(self, m): + self._log(DEBUG, 'Global request denied.') + self.global_response = None + if self.completion_event is not None: + self.completion_event.set() + + def _parse_channel_open_success(self, m): + chanid = m.get_int() + server_chanid = m.get_int() + server_window_size = m.get_int() + server_max_packet_size = m.get_int() + chan = self._channels.get(chanid) + if chan is None: + self._log(WARNING, 'Success for unrequested channel! [??]') + return + self.lock.acquire() + try: + chan._set_remote_channel(server_chanid, server_window_size, server_max_packet_size) + self._log(INFO, 'Secsh channel %d opened.' % chanid) + if chanid in self.channel_events: + self.channel_events[chanid].set() + del self.channel_events[chanid] + finally: + self.lock.release() + return + + def _parse_channel_open_failure(self, m): + chanid = m.get_int() + reason = m.get_int() + reason_str = m.get_string() + lang = m.get_string() + reason_text = CONNECTION_FAILED_CODE.get(reason, '(unknown code)') + self._log(INFO, 'Secsh channel %d open FAILED: %s: %s' % (chanid, reason_str, reason_text)) + self.lock.acquire() + try: + self.saved_exception = ChannelException(reason, reason_text) + if chanid in self.channel_events: + self._channels.delete(chanid) + if chanid in self.channel_events: + self.channel_events[chanid].set() + del self.channel_events[chanid] + finally: + self.lock.release() + return + + def _parse_channel_open(self, m): + kind = m.get_string() + chanid = m.get_int() + initial_window_size = m.get_int() + max_packet_size = m.get_int() + reject = False + if (kind == 'auth-agent@openssh.com') and (self._forward_agent_handler is not None): + self._log(DEBUG, 'Incoming forward agent connection') + self.lock.acquire() + try: + my_chanid = self._next_channel() + finally: + self.lock.release() + elif (kind == 'x11') and (self._x11_handler is not None): + origin_addr = m.get_string() + origin_port = m.get_int() + self._log(DEBUG, 'Incoming x11 connection from %s:%d' % (origin_addr, origin_port)) + self.lock.acquire() + try: + my_chanid = self._next_channel() + finally: + self.lock.release() + elif (kind == 'forwarded-tcpip') and (self._tcp_handler is not None): + server_addr = m.get_string() + server_port = m.get_int() + origin_addr = m.get_string() + origin_port = m.get_int() + self._log(DEBUG, 'Incoming tcp forwarded connection from %s:%d' % (origin_addr, origin_port)) + self.lock.acquire() + try: + my_chanid = self._next_channel() + finally: + self.lock.release() + elif not self.server_mode: + self._log(DEBUG, 'Rejecting "%s" channel request from server.' % kind) + reject = True + reason = OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED + else: + self.lock.acquire() + try: + my_chanid = self._next_channel() + finally: + self.lock.release() + if kind == 'direct-tcpip': + # handle direct-tcpip requests comming from the client + dest_addr = m.get_string() + dest_port = m.get_int() + origin_addr = m.get_string() + origin_port = m.get_int() + reason = self.server_object.check_channel_direct_tcpip_request( + my_chanid, (origin_addr, origin_port), + (dest_addr, dest_port)) + else: + reason = self.server_object.check_channel_request(kind, my_chanid) + if reason != OPEN_SUCCEEDED: + self._log(DEBUG, 'Rejecting "%s" channel request from client.' % kind) + reject = True + if reject: + msg = Message() + msg.add_byte(chr(MSG_CHANNEL_OPEN_FAILURE)) + msg.add_int(chanid) + msg.add_int(reason) + msg.add_string('') + msg.add_string('en') + self._send_message(msg) + return + + chan = Channel(my_chanid) + self.lock.acquire() + try: + self._channels.put(my_chanid, chan) + self.channels_seen[my_chanid] = True + chan._set_transport(self) + chan._set_window(self.window_size, self.max_packet_size) + chan._set_remote_channel(chanid, initial_window_size, max_packet_size) + finally: + self.lock.release() + m = Message() + m.add_byte(chr(MSG_CHANNEL_OPEN_SUCCESS)) + m.add_int(chanid) + m.add_int(my_chanid) + m.add_int(self.window_size) + m.add_int(self.max_packet_size) + self._send_message(m) + self._log(INFO, 'Secsh channel %d (%s) opened.', my_chanid, kind) + if kind == 'auth-agent@openssh.com': + self._forward_agent_handler(chan) + elif kind == 'x11': + self._x11_handler(chan, (origin_addr, origin_port)) + elif kind == 'forwarded-tcpip': + chan.origin_addr = (origin_addr, origin_port) + self._tcp_handler(chan, (origin_addr, origin_port), (server_addr, server_port)) + else: + self._queue_incoming_channel(chan) + + def _parse_debug(self, m): + always_display = m.get_boolean() + msg = m.get_string() + lang = m.get_string() + self._log(DEBUG, 'Debug msg: ' + util.safe_string(msg)) + + def _get_subsystem_handler(self, name): + try: + self.lock.acquire() + if name not in self.subsystem_table: + return (None, [], {}) + return self.subsystem_table[name] + finally: + self.lock.release() + + _handler_table = { + MSG_NEWKEYS: _parse_newkeys, + MSG_GLOBAL_REQUEST: _parse_global_request, + MSG_REQUEST_SUCCESS: _parse_request_success, + MSG_REQUEST_FAILURE: _parse_request_failure, + MSG_CHANNEL_OPEN_SUCCESS: _parse_channel_open_success, + MSG_CHANNEL_OPEN_FAILURE: _parse_channel_open_failure, + MSG_CHANNEL_OPEN: _parse_channel_open, + MSG_KEXINIT: _negotiate_keys, + } + + _channel_handler_table = { + MSG_CHANNEL_SUCCESS: Channel._request_success, + MSG_CHANNEL_FAILURE: Channel._request_failed, + MSG_CHANNEL_DATA: Channel._feed, + MSG_CHANNEL_EXTENDED_DATA: Channel._feed_extended, + MSG_CHANNEL_WINDOW_ADJUST: Channel._window_adjust, + MSG_CHANNEL_REQUEST: Channel._handle_request, + MSG_CHANNEL_EOF: Channel._handle_eof, + MSG_CHANNEL_CLOSE: Channel._handle_close, + } diff --git a/contrib/site-packages/paramiko/util.py b/contrib/site-packages/paramiko/util.py new file mode 100644 index 00000000..85ee6b06 --- /dev/null +++ b/contrib/site-packages/paramiko/util.py @@ -0,0 +1,311 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +Useful functions used by the rest of paramiko. +""" + +from __future__ import generators + +import array +from binascii import hexlify, unhexlify +import errno +import sys +import struct +import traceback +import threading + +from paramiko.common import * +from paramiko.config import SSHConfig + + +# Change by RogerB - python < 2.3 doesn't have enumerate so we implement it +if sys.version_info < (2,3): + class enumerate: + def __init__ (self, sequence): + self.sequence = sequence + def __iter__ (self): + count = 0 + for item in self.sequence: + yield (count, item) + count += 1 + + +def inflate_long(s, always_positive=False): + "turns a normalized byte string into a long-int (adapted from Crypto.Util.number)" + out = 0L + negative = 0 + if not always_positive and (len(s) > 0) and (ord(s[0]) >= 0x80): + negative = 1 + if len(s) % 4: + filler = '\x00' + if negative: + filler = '\xff' + s = filler * (4 - len(s) % 4) + s + for i in range(0, len(s), 4): + out = (out << 32) + struct.unpack('>I', s[i:i+4])[0] + if negative: + out -= (1L << (8 * len(s))) + return out + +def deflate_long(n, add_sign_padding=True): + "turns a long-int into a normalized byte string (adapted from Crypto.Util.number)" + # after much testing, this algorithm was deemed to be the fastest + s = '' + n = long(n) + while (n != 0) and (n != -1): + s = struct.pack('>I', n & 0xffffffffL) + s + n = n >> 32 + # strip off leading zeros, FFs + for i in enumerate(s): + if (n == 0) and (i[1] != '\000'): + break + if (n == -1) and (i[1] != '\xff'): + break + else: + # degenerate case, n was either 0 or -1 + i = (0,) + if n == 0: + s = '\000' + else: + s = '\xff' + s = s[i[0]:] + if add_sign_padding: + if (n == 0) and (ord(s[0]) >= 0x80): + s = '\x00' + s + if (n == -1) and (ord(s[0]) < 0x80): + s = '\xff' + s + return s + +def format_binary_weird(data): + out = '' + for i in enumerate(data): + out += '%02X' % ord(i[1]) + if i[0] % 2: + out += ' ' + if i[0] % 16 == 15: + out += '\n' + return out + +def format_binary(data, prefix=''): + x = 0 + out = [] + while len(data) > x + 16: + out.append(format_binary_line(data[x:x+16])) + x += 16 + if x < len(data): + out.append(format_binary_line(data[x:])) + return [prefix + x for x in out] + +def format_binary_line(data): + left = ' '.join(['%02X' % ord(c) for c in data]) + right = ''.join([('.%c..' % c)[(ord(c)+63)//95] for c in data]) + return '%-50s %s' % (left, right) + +def hexify(s): + return hexlify(s).upper() + +def unhexify(s): + return unhexlify(s) + +def safe_string(s): + out = '' + for c in s: + if (ord(c) >= 32) and (ord(c) <= 127): + out += c + else: + out += '%%%02X' % ord(c) + return out + +# ''.join([['%%%02X' % ord(c), c][(ord(c) >= 32) and (ord(c) <= 127)] for c in s]) + +def bit_length(n): + norm = deflate_long(n, 0) + hbyte = ord(norm[0]) + if hbyte == 0: + return 1 + bitlen = len(norm) * 8 + while not (hbyte & 0x80): + hbyte <<= 1 + bitlen -= 1 + return bitlen + +def tb_strings(): + return ''.join(traceback.format_exception(*sys.exc_info())).split('\n') + +def generate_key_bytes(hashclass, salt, key, nbytes): + """ + Given a password, passphrase, or other human-source key, scramble it + through a secure hash into some keyworthy bytes. This specific algorithm + is used for encrypting/decrypting private key files. + + @param hashclass: class from L{Crypto.Hash} that can be used as a secure + hashing function (like C{MD5} or C{SHA}). + @type hashclass: L{Crypto.Hash} + @param salt: data to salt the hash with. + @type salt: string + @param key: human-entered password or passphrase. + @type key: string + @param nbytes: number of bytes to generate. + @type nbytes: int + @return: key data + @rtype: string + """ + keydata = '' + digest = '' + if len(salt) > 8: + salt = salt[:8] + while nbytes > 0: + hash_obj = hashclass.new() + if len(digest) > 0: + hash_obj.update(digest) + hash_obj.update(key) + hash_obj.update(salt) + digest = hash_obj.digest() + size = min(nbytes, len(digest)) + keydata += digest[:size] + nbytes -= size + return keydata + +def load_host_keys(filename): + """ + Read a file of known SSH host keys, in the format used by openssh, and + return a compound dict of C{hostname -> keytype ->} L{PKey }. + The hostname may be an IP address or DNS name. The keytype will be either + C{"ssh-rsa"} or C{"ssh-dss"}. + + This type of file unfortunately doesn't exist on Windows, but on posix, + it will usually be stored in C{os.path.expanduser("~/.ssh/known_hosts")}. + + Since 1.5.3, this is just a wrapper around L{HostKeys}. + + @param filename: name of the file to read host keys from + @type filename: str + @return: dict of host keys, indexed by hostname and then keytype + @rtype: dict(hostname, dict(keytype, L{PKey })) + """ + from paramiko.hostkeys import HostKeys + return HostKeys(filename) + +def parse_ssh_config(file_obj): + """ + Provided only as a backward-compatible wrapper around L{SSHConfig}. + """ + config = SSHConfig() + config.parse(file_obj) + return config + +def lookup_ssh_host_config(hostname, config): + """ + Provided only as a backward-compatible wrapper around L{SSHConfig}. + """ + return config.lookup(hostname) + +def mod_inverse(x, m): + # it's crazy how small python can make this function. + u1, u2, u3 = 1, 0, m + v1, v2, v3 = 0, 1, x + + while v3 > 0: + q = u3 // v3 + u1, v1 = v1, u1 - v1 * q + u2, v2 = v2, u2 - v2 * q + u3, v3 = v3, u3 - v3 * q + if u2 < 0: + u2 += m + return u2 + +_g_thread_ids = {} +_g_thread_counter = 0 +_g_thread_lock = threading.Lock() +def get_thread_id(): + global _g_thread_ids, _g_thread_counter, _g_thread_lock + tid = id(threading.currentThread()) + try: + return _g_thread_ids[tid] + except KeyError: + _g_thread_lock.acquire() + try: + _g_thread_counter += 1 + ret = _g_thread_ids[tid] = _g_thread_counter + finally: + _g_thread_lock.release() + return ret + +def log_to_file(filename, level=DEBUG): + "send paramiko logs to a logfile, if they're not already going somewhere" + l = logging.getLogger("paramiko") + if len(l.handlers) > 0: + return + l.setLevel(level) + f = open(filename, 'w') + lh = logging.StreamHandler(f) + lh.setFormatter(logging.Formatter('%(levelname)-.3s [%(asctime)s.%(msecs)03d] thr=%(_threadid)-3d %(name)s: %(message)s', + '%Y%m%d-%H:%M:%S')) + l.addHandler(lh) + +# make only one filter object, so it doesn't get applied more than once +class PFilter (object): + def filter(self, record): + record._threadid = get_thread_id() + return True +_pfilter = PFilter() + +def get_logger(name): + l = logging.getLogger(name) + l.addFilter(_pfilter) + return l + +def retry_on_signal(function): + """Retries function until it doesn't raise an EINTR error""" + while True: + try: + return function() + except EnvironmentError, e: + if e.errno != errno.EINTR: + raise + +class Counter (object): + """Stateful counter for CTR mode crypto""" + def __init__(self, nbits, initial_value=1L, overflow=0L): + self.blocksize = nbits / 8 + self.overflow = overflow + # start with value - 1 so we don't have to store intermediate values when counting + # could the iv be 0? + if initial_value == 0: + self.value = array.array('c', '\xFF' * self.blocksize) + else: + x = deflate_long(initial_value - 1, add_sign_padding=False) + self.value = array.array('c', '\x00' * (self.blocksize - len(x)) + x) + + def __call__(self): + """Increament the counter and return the new value""" + i = self.blocksize - 1 + while i > -1: + c = self.value[i] = chr((ord(self.value[i]) + 1) % 256) + if c != '\x00': + return self.value.tostring() + i -= 1 + # counter reset + x = deflate_long(self.overflow, add_sign_padding=False) + self.value = array.array('c', '\x00' * (self.blocksize - len(x)) + x) + return self.value.tostring() + + def new(cls, nbits, initial_value=1L, overflow=0L): + return cls(nbits, initial_value=initial_value, overflow=overflow) + new = classmethod(new) diff --git a/contrib/site-packages/paramiko/win_pageant.py b/contrib/site-packages/paramiko/win_pageant.py new file mode 100644 index 00000000..de1cd64b --- /dev/null +++ b/contrib/site-packages/paramiko/win_pageant.py @@ -0,0 +1,125 @@ +# Copyright (C) 2005 John Arbash-Meinel +# Modified up by: Todd Whiteman +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + +""" +Functions for communicating with Pageant, the basic windows ssh agent program. +""" + +from __future__ import with_statement + +import struct +import threading +import array +import platform +import ctypes.wintypes + +from . import _winapi + +_AGENT_COPYDATA_ID = 0x804e50ba +_AGENT_MAX_MSGLEN = 8192 +# Note: The WM_COPYDATA value is pulled from win32con, as a workaround +# so we do not have to import this huge library just for this one variable. +win32con_WM_COPYDATA = 74 + + +def _get_pageant_window_object(): + return ctypes.windll.user32.FindWindowA('Pageant', 'Pageant') + + +def can_talk_to_agent(): + """ + Check to see if there is a "Pageant" agent we can talk to. + + This checks both if we have the required libraries (win32all or ctypes) + and if there is a Pageant currently running. + """ + return bool(_get_pageant_window_object()) + +ULONG_PTR = ctypes.c_uint64 if platform.architecture()[0] == '64bit' else ctypes.c_uint32 +class COPYDATASTRUCT(ctypes.Structure): + """ + ctypes implementation of + http://msdn.microsoft.com/en-us/library/windows/desktop/ms649010%28v=vs.85%29.aspx + """ + _fields_ = [ + ('num_data', ULONG_PTR), + ('data_size', ctypes.wintypes.DWORD), + ('data_loc', ctypes.c_void_p), + ] + +def _query_pageant(msg): + """ + Communication with the Pageant process is done through a shared + memory-mapped file. + """ + hwnd = _get_pageant_window_object() + if not hwnd: + # Raise a failure to connect exception, pageant isn't running anymore! + return None + + # create a name for the mmap + map_name = 'PageantRequest%08x' % threading.current_thread().ident + + pymap = _winapi.MemoryMap(map_name, _AGENT_MAX_MSGLEN, + _winapi.get_security_attributes_for_user(), + ) + with pymap: + pymap.write(msg) + # Create an array buffer containing the mapped filename + char_buffer = array.array("c", map_name + '\0') + char_buffer_address, char_buffer_size = char_buffer.buffer_info() + # Create a string to use for the SendMessage function call + cds = COPYDATASTRUCT(_AGENT_COPYDATA_ID, char_buffer_size, + char_buffer_address) + + response = ctypes.windll.user32.SendMessageA(hwnd, + win32con_WM_COPYDATA, ctypes.sizeof(cds), ctypes.byref(cds)) + + if response > 0: + pymap.seek(0) + datalen = pymap.read(4) + retlen = struct.unpack('>I', datalen)[0] + return datalen + pymap.read(retlen) + return None + +class PageantConnection (object): + """ + Mock "connection" to an agent which roughly approximates the behavior of + a unix local-domain socket (as used by Agent). Requests are sent to the + pageant daemon via special Windows magick, and responses are buffered back + for subsequent reads. + """ + + def __init__(self): + self._response = None + + def send(self, data): + self._response = _query_pageant(data) + + def recv(self, n): + if self._response is None: + return '' + ret = self._response[:n] + self._response = self._response[n:] + if self._response == '': + self._response = None + return ret + + def close(self): + pass From 63b956f03bc6826ea2c69effe7209792e97771ca Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Thu, 24 Apr 2014 16:26:04 -0400 Subject: [PATCH 02/50] Remove outdated paper tests (#18) --- tests/paper/functions.sh | 113 -------------------------------- tests/paper/local-ib-null.sh | 6 -- tests/paper/local-iperf-null.sh | 6 -- tests/paper/local-tcp-null.sh | 6 -- tests/paper/local-xtcp-null.sh | 6 -- tests/paper/localdriver.sh | 21 ------ tests/paper/remotedriver.sh | 21 ------ 7 files changed, 179 deletions(-) delete mode 100755 tests/paper/local-ib-null.sh delete mode 100755 tests/paper/local-iperf-null.sh delete mode 100755 tests/paper/local-tcp-null.sh delete mode 100755 tests/paper/local-xtcp-null.sh delete mode 100755 tests/paper/localdriver.sh delete mode 100755 tests/paper/remotedriver.sh diff --git a/tests/paper/functions.sh b/tests/paper/functions.sh index b6fbfaab..64767776 100755 --- a/tests/paper/functions.sh +++ b/tests/paper/functions.sh @@ -9,119 +9,6 @@ abort () { exit 101 } -#TODO: NUMA -#TODO: congestion -# Arguments: transport [bytes] -# transport can be tcp, xtcp, or ib -run_local_xdd () { - # override configuration values - E2ESRC=localhost - E2EDEST=localhost - - # set option variables - if [ "$1" = 'tcp' ] - then - local XNIOPT='' - elif [ "$1" = 'xtcp' ] - then - local XNIOPT='-xni tcp' - elif [ "$1" = 'ib' ] - then - local XNIOPT='-xni ib' - local IBDEVICEOPT="-ibdevice ${IBDEVICE}" - else - abort "unknown transport '$1'" - fi - - if [ -n "$2" ] - then - local BYTES="$2" - fi - - local TARGETOPT="-targets 1 null" # reads/writes are no-ops - local E2EOPT="-e2e dest ${E2ESRC}:${E2EPORT},${E2ETHREADS}" - local REQSIZEOPT="-reqsize ${REQSIZE}" - local BYTESOPT="-bytes ${BYTES}" - - # start destination side in background and ignore output - ${XDD} \ - ${XNIOPT} \ - ${IBDEVICEOPT} \ - ${TARGETOPT} \ - -op write -e2e isdest \ - ${E2EOPT} \ - ${BYTESOPT} \ - ${REQSIZEOPT} \ - >/dev/null \ - & - - # wait for destination side to start - sleep 3 - - # start source side and output - # pass,target,queue,size,ops,elapsed,bandwidth,iops,latency,cpu,op,reqsize - ${XDD} \ - ${XNIOPT} \ - ${IBDEVICEOPT} \ - ${TARGETOPT} \ - -op read -e2e issrc \ - ${E2EOPT} \ - ${BYTESOPT} \ - ${REQSIZEOPT} \ - | grep -o 'COMBINED .*' \ - | sed 's/ */,/g' \ - | cut -d ',' -f 2-13 -} - -#TODO: NUMA -#TODO: congestion -# Arguments: [bytes] -run_local_iperf () { - # override configuration values - E2ESRC=localhost - E2EDEST=localhost - - # set option variables - if [ -n "$1" ] - then - local BYTES="$1" - fi - - local CLIENTOPT="-c ${E2EDEST}" - local CSVOPT="-y c" - local INTERVALOPT="-i 3600" # large interval so only the total is output - local BUFLENOPT="-l $((${REQSIZE}*1024))" - local NUMOPT="-n ${BYTES}" - - # start destination side in background and ignore output - ${IPERF} \ - -s \ - >/dev/null 2>/dev/null \ - & - - # wait for destination side to start - sleep 3 - - # start source side and save output - local CSVOUT=$( - ${IPERF} \ - ${CLIENTOPT} \ - ${CSVOPT} \ - ${INTERVALOPT} \ - ${BUFLENOPT} \ - ${NUMOPT} \ - | tail -1 - ) - - # parse output values - local ELAPSEDOUT=`cut -d , -f 7 <<<${CSVOUT} | cut -d - -f 2` - local BANDWIDTHOUT=$((`cut -d , -f 9 <<<${CSVOUT}`/8000000)) - - # output - # pass,target,queue,size,ops,elapsed,bandwidth,iops,latency,cpu,op,reqsize - echo ,,,${BYTES},,${ELAPSEDOUT},${BANDWIDTHOUT},,,,,${REQSIZE} -} - # Arguments: transport [bytes] # transport can be tcp, xtcp, or ib run_remote_xdd () { diff --git a/tests/paper/local-ib-null.sh b/tests/paper/local-ib-null.sh deleted file mode 100755 index 90e6ad91..00000000 --- a/tests/paper/local-ib-null.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# load support functions -source ./functions.sh - -run_local_xdd ib diff --git a/tests/paper/local-iperf-null.sh b/tests/paper/local-iperf-null.sh deleted file mode 100755 index 4003d21b..00000000 --- a/tests/paper/local-iperf-null.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# load support functions -source ./functions.sh - -run_local_iperf diff --git a/tests/paper/local-tcp-null.sh b/tests/paper/local-tcp-null.sh deleted file mode 100755 index ed754f99..00000000 --- a/tests/paper/local-tcp-null.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# load support functions -source ./functions.sh - -run_local_xdd tcp diff --git a/tests/paper/local-xtcp-null.sh b/tests/paper/local-xtcp-null.sh deleted file mode 100755 index 9a1f8fe8..00000000 --- a/tests/paper/local-xtcp-null.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# load support functions -source ./functions.sh - -run_local_xdd xtcp diff --git a/tests/paper/localdriver.sh b/tests/paper/localdriver.sh deleted file mode 100755 index 9a7f356b..00000000 --- a/tests/paper/localdriver.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -NITER=10 - -TCPOUT=tcp-10g.csv -XTCPOUT=xtcp-10g.csv -IPERFOUT=iperf-10g.csv - -HEADER="pass,target,queue,size,ops,elapsed,bandwidth,iops,latency,cpu,op,reqsize" -echo ${HEADER} >${TCPOUT} -echo ${HEADER} >${XTCPOUT} -echo ${HEADER} >${IPERFOUT} -for i in `seq $NITER` -do - ./local-tcp-null.sh >>${TCPOUT} - sleep 4 - ./local-xtcp-null.sh >>${XTCPOUT} - sleep 4 - ./local-iperf-null.sh >>${IPERFOUT} - sleep 4 -done diff --git a/tests/paper/remotedriver.sh b/tests/paper/remotedriver.sh deleted file mode 100755 index 32193f39..00000000 --- a/tests/paper/remotedriver.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -NITER=10 - -TCPOUT=tcp-10g.csv -XTCPOUT=xtcp-10g.csv -IPERFOUT=iperf-10g.csv - -HEADER="pass,target,queue,size,ops,elapsed,bandwidth,iops,latency,cpu,op,reqsize" -echo ${HEADER} >${TCPOUT} -echo ${HEADER} >${XTCPOUT} -echo ${HEADER} >${IPERFOUT} -for i in `seq $NITER` -do - ./remote-tcp-null.sh >>${TCPOUT} - sleep 4 - ./remote-xtcp-null.sh >>${XTCPOUT} - sleep 4 - ./remote-iperf-null.sh >>${IPERFOUT} - sleep 4 -done From 950c5a38855d556f0654d57a089f39e122b87fa8 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Thu, 24 Apr 2014 16:39:18 -0400 Subject: [PATCH 03/50] Make the Iperf port configurable (#18) --- tests/paper/config.sh | 3 +++ tests/paper/functions.sh | 3 +++ 2 files changed, 6 insertions(+) diff --git a/tests/paper/config.sh b/tests/paper/config.sh index 1c962427..9ab2b9a8 100755 --- a/tests/paper/config.sh +++ b/tests/paper/config.sh @@ -49,6 +49,9 @@ E2EPORT=40010 # number of network threads (also number of I/O threads) E2ETHREADS=1 +# TCP port for iperf to listen on/connect to +IPERFPORT=40310 + # device to use for XNI InfiniBand IBDEVICE="mlx4_0" diff --git a/tests/paper/functions.sh b/tests/paper/functions.sh index 64767776..fa9ef83e 100755 --- a/tests/paper/functions.sh +++ b/tests/paper/functions.sh @@ -102,6 +102,7 @@ run_remote_iperf () { local INTERVALOPT="-i 3600" # large interval so only the total is output local BUFLENOPT="-l $((${REQSIZE}*1024))" local NUMOPT="-n ${BYTES}" + local PORTOPT="-p ${IPERFPORT}" [ -n "${CONGESTION}" ] && local CONGESTIONOPT="-Z ${CONGESTION}" local SSHOPT="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"\ " -o BatchMode=yes" @@ -110,6 +111,7 @@ run_remote_iperf () { ${NUMACMD} \ ${IPERF} \ -s \ + ${PORTOPT} \ ${CONGESTIONOPT} \ >/dev/null 2>/dev/null \ & @@ -125,6 +127,7 @@ run_remote_iperf () { ${NUMACMD} \ ${IPERF} \ ${CLIENTOPT} \ + ${PORTOPT} \ ${CSVOPT} \ ${INTERVALOPT} \ ${BUFLENOPT} \ From 8e97a8d24378a38931c99cd7623cc21031f8e6c8 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Thu, 24 Apr 2014 16:42:32 -0400 Subject: [PATCH 04/50] Optionally set request size from test function argument (#18) --- tests/paper/functions.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/paper/functions.sh b/tests/paper/functions.sh index fa9ef83e..2b1ce3d1 100755 --- a/tests/paper/functions.sh +++ b/tests/paper/functions.sh @@ -9,7 +9,7 @@ abort () { exit 101 } -# Arguments: transport [bytes] +# Arguments: transport [bytes] [reqsize] # transport can be tcp, xtcp, or ib run_remote_xdd () { # set option variables @@ -38,6 +38,11 @@ run_remote_xdd () { local BYTES="$2" fi + if [ -n "$3" ] + then + local REQSIZE="$3" + fi + local TARGETOPT="-targets 1 null" # reads/writes are no-ops local E2EOPT="-e2e dest ${E2EDEST}:${E2EPORT},${E2ETHREADS}" [ -n "${CONGESTION}" ] && local CONGESTIONOPT="-congestion ${CONGESTION}" @@ -83,7 +88,7 @@ run_remote_xdd () { | cut -d ',' -f 2-13 } -# Arguments: [bytes] +# Arguments: [bytes] [reqsize] run_remote_iperf () { # set option variables local NUMACMD="" @@ -97,6 +102,11 @@ run_remote_iperf () { local BYTES="$1" fi + if [ -n "$2" ] + then + local REQSIZE="$2" + fi + local CLIENTOPT="-c ${E2EDEST}" local CSVOPT="-y c" local INTERVALOPT="-i 3600" # large interval so only the total is output From b6bf762d69f52e63d338033c15ad41661c7461ee Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Thu, 24 Apr 2014 16:46:42 -0400 Subject: [PATCH 05/50] Add some test scripts for the XNI paper (#18) --- tests/paper/drive.sh | 35 ++++++++++++++++++++++++++++++++ tests/paper/numadrive.sh | 38 +++++++++++++++++++++++++++++++++++ tests/paper/remote-ib-null.sh | 6 ++++++ 3 files changed, 79 insertions(+) create mode 100755 tests/paper/drive.sh create mode 100755 tests/paper/numadrive.sh create mode 100755 tests/paper/remote-ib-null.sh diff --git a/tests/paper/drive.sh b/tests/paper/drive.sh new file mode 100755 index 00000000..f0abe051 --- /dev/null +++ b/tests/paper/drive.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +source ./functions.sh + +declare -r ONEGB=1000000000 +declare -r TENGB=$((10*${ONEGB})) +declare -r TWENTYGB=$((20*${ONEGB})) +declare -r FORTYGB=$((40*${ONEGB})) +declare -r EIGHTYGB=$((80*${ONEGB})) + +NITER=10 + +OUTFILE=results.csv + +HEADER="proto,pass,target,queue,size,ops,elapsed,bandwidth,iops,latency,cpu,op,reqsize" + +echo ${HEADER} >${OUTFILE} + +for size in ${TENGB} ${TWENTYGB} ${FORTYGB} ${EIGHTYGB} +do + for i in `seq ${NITER}` + do + echo -n "iperf," >>${OUTFILE} + run_remote_iperf ${size} >>${OUTFILE} + sleep 4 + + echo -n "tcp," >>${OUTFILE} + run_remote_xdd tcp ${size} >>${OUTFILE} + sleep 4 + + echo -n "xtcp," >>${OUTFILE} + run_remote_xdd xtcp ${size} >>${OUTFILE} + sleep 4 + done +done diff --git a/tests/paper/numadrive.sh b/tests/paper/numadrive.sh new file mode 100755 index 00000000..6223145f --- /dev/null +++ b/tests/paper/numadrive.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +source ./functions.sh + +declare -r ONEGB=1000000000 +declare -r TENGB=$((10*${ONEGB})) +declare -r TWENTYGB=$((20*${ONEGB})) +declare -r FORTYGB=$((40*${ONEGB})) +declare -r EIGHTYGB=$((80*${ONEGB})) + +NITER=10 + +OUTFILE="numa-node" +if [ "$NUMA" == 'true' ] +then + OUTFILE="${OUTFILE}${NUMANODE}" +else + OUTFILE="${OUTFILE}default" +fi +OUTFILE="${OUTFILE}-${E2ETHREADS}stream.csv" + +HEADER="proto,pass,target,queue,size,ops,elapsed,bandwidth,iops,latency,cpu,op,reqsize" + +echo ${HEADER} >${OUTFILE} + +for reqsize in 1024 2048 4096 8192 16384 +do + for i in `seq ${NITER}` + do + echo -n "iperf," >>${OUTFILE} + run_remote_iperf ${EIGHTYGB} ${reqsize} >>${OUTFILE} + sleep 4 + + echo -n "xtcp," >>${OUTFILE} + run_remote_xdd xtcp ${EIGHTYGB} ${reqsize} >>${OUTFILE} + sleep 4 + done +done diff --git a/tests/paper/remote-ib-null.sh b/tests/paper/remote-ib-null.sh new file mode 100755 index 00000000..295e4d0d --- /dev/null +++ b/tests/paper/remote-ib-null.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# load support functions +source ./functions.sh + +run_remote_xdd ib From c783025ae8094c41fa412dec07fbc4e54fe6359c Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Thu, 24 Apr 2014 20:54:44 -0400 Subject: [PATCH 06/50] Require iperf 2.0.5 for paper tests --- tests/paper/functions.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/paper/functions.sh b/tests/paper/functions.sh index 2b1ce3d1..2ff6c9e4 100755 --- a/tests/paper/functions.sh +++ b/tests/paper/functions.sh @@ -90,6 +90,15 @@ run_remote_xdd () { # Arguments: [bytes] [reqsize] run_remote_iperf () { + # ensure that Iperf can understand our command line and that we + # can understand Iperf's output + local VERSION=`iperf --version 2>&1 | cut -d ' ' -f 1-3` + if [ "$VERSION" != 'iperf version 2.0.5' ] + then + abort "unsupported iperf: ${VERSION}" + fi + + # set option variables local NUMACMD="" if [ "$NUMA" == 'true' ] From a7cfe3d5d1d8be29636eaf20edb167a192115ae4 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Thu, 24 Apr 2014 20:57:31 -0400 Subject: [PATCH 07/50] Fix iperf options The interval option used here breaks with multiple threads. Using a large interval value was just a (possibly unnecessary) hack anyway. --- tests/paper/functions.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/paper/functions.sh b/tests/paper/functions.sh index 2ff6c9e4..d5619657 100755 --- a/tests/paper/functions.sh +++ b/tests/paper/functions.sh @@ -118,7 +118,6 @@ run_remote_iperf () { local CLIENTOPT="-c ${E2EDEST}" local CSVOPT="-y c" - local INTERVALOPT="-i 3600" # large interval so only the total is output local BUFLENOPT="-l $((${REQSIZE}*1024))" local NUMOPT="-n ${BYTES}" local PORTOPT="-p ${IPERFPORT}" @@ -148,7 +147,6 @@ run_remote_iperf () { ${CLIENTOPT} \ ${PORTOPT} \ ${CSVOPT} \ - ${INTERVALOPT} \ ${BUFLENOPT} \ ${NUMOPT} \ ${CONGESTIONOPT} \ From 8a399318655280984c1d7838d85918e6726a9e32 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Thu, 24 Apr 2014 21:42:38 -0400 Subject: [PATCH 08/50] Support multiple threads in iperf paper tests (#18) --- tests/paper/functions.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/paper/functions.sh b/tests/paper/functions.sh index d5619657..15842077 100755 --- a/tests/paper/functions.sh +++ b/tests/paper/functions.sh @@ -88,6 +88,13 @@ run_remote_xdd () { | cut -d ',' -f 2-13 } +function ceiling() { + local dividend=$1 + local divisor=$2 + local result=$(( ((${dividend} - (${dividend} % ${divisor}))/${divisor}) + 1 )) + echo ${result} +} + # Arguments: [bytes] [reqsize] run_remote_iperf () { # ensure that Iperf can understand our command line and that we @@ -119,8 +126,9 @@ run_remote_iperf () { local CLIENTOPT="-c ${E2EDEST}" local CSVOPT="-y c" local BUFLENOPT="-l $((${REQSIZE}*1024))" - local NUMOPT="-n ${BYTES}" + local NUMOPT="-n `ceiling ${BYTES} ${E2ETHREADS}`" local PORTOPT="-p ${IPERFPORT}" + local PARALLELOPT="-P ${E2ETHREADS}" [ -n "${CONGESTION}" ] && local CONGESTIONOPT="-Z ${CONGESTION}" local SSHOPT="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"\ " -o BatchMode=yes" @@ -149,6 +157,7 @@ run_remote_iperf () { ${CSVOPT} \ ${BUFLENOPT} \ ${NUMOPT} \ + ${PARALLELOPT} \ ${CONGESTIONOPT} \ | tail -1 ) From d747dc5f83c67349159977862d63661a06077963 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Thu, 24 Apr 2014 22:18:49 -0400 Subject: [PATCH 09/50] Cleanup iperf server processes (#18) Before this commit a test run could leave as many as 100 (!) iperf servers running. --- tests/paper/functions.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/paper/functions.sh b/tests/paper/functions.sh index 15842077..533b73a7 100755 --- a/tests/paper/functions.sh +++ b/tests/paper/functions.sh @@ -141,6 +141,7 @@ run_remote_iperf () { ${CONGESTIONOPT} \ >/dev/null 2>/dev/null \ & + local IPERFPID=$! # wait for destination side to start sleep 3 @@ -169,4 +170,7 @@ run_remote_iperf () { # output # pass,target,queue,size,ops,elapsed,bandwidth,iops,latency,cpu,op,reqsize echo ,,,${BYTES},,${ELAPSEDOUT},${BANDWIDTHOUT},,,,,${REQSIZE} + + # cleanup by killing iperf server + kill "${IPERFPID}" } From f85376aa6a974bbf7bab6a141ef9054b8cc9df15 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Fri, 25 Apr 2014 01:53:36 -0400 Subject: [PATCH 10/50] Use the correct iperf command --- tests/paper/functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/paper/functions.sh b/tests/paper/functions.sh index 533b73a7..c696c8ee 100755 --- a/tests/paper/functions.sh +++ b/tests/paper/functions.sh @@ -99,7 +99,7 @@ function ceiling() { run_remote_iperf () { # ensure that Iperf can understand our command line and that we # can understand Iperf's output - local VERSION=`iperf --version 2>&1 | cut -d ' ' -f 1-3` + local VERSION=`${IPERF} --version 2>&1 | cut -d ' ' -f 1-3` if [ "$VERSION" != 'iperf version 2.0.5' ] then abort "unsupported iperf: ${VERSION}" From 9273dd66c4b61e55e3b44e13aab4dc04e1b38831 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Fri, 25 Apr 2014 01:54:01 -0400 Subject: [PATCH 11/50] Fix division bug in test script --- tests/paper/functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/paper/functions.sh b/tests/paper/functions.sh index c696c8ee..88b21033 100755 --- a/tests/paper/functions.sh +++ b/tests/paper/functions.sh @@ -91,7 +91,7 @@ run_remote_xdd () { function ceiling() { local dividend=$1 local divisor=$2 - local result=$(( ((${dividend} - (${dividend} % ${divisor}))/${divisor}) + 1 )) + local result=$(( (${dividend} - 1)/${divisor} + 1)) echo ${result} } From 1a8b45dad58a0e637b8aa323084dc29c85c4a4ab Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Tue, 29 Apr 2014 20:20:23 -0400 Subject: [PATCH 12/50] Set the socket buffer sizes in XNI (#18) TODO: make the sizes configurable. --- src/xni/xni_tcp.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/xni/xni_tcp.c b/src/xni/xni_tcp.c index 85525097..0cdc9bfb 100644 --- a/src/xni/xni_tcp.c +++ b/src/xni/xni_tcp.c @@ -226,6 +226,28 @@ static int tcp_accept_connection(xni_context_t ctx_, struct xni_endpoint* local, #endif // HAVE_DECL_TCP_CONGESTION } + //TODO: replace with a real value + optval = 131072; + int rc = setsockopt(servers[i], + SOL_SOCKET, + SO_SNDBUF, + &optval, + sizeof(optval)); + if (rc) { + perror("setsockopt"); + } + + //TODO: replace with a real value + optval = 131072; + rc = setsockopt(servers[i], + SOL_SOCKET, + SO_RCVBUF, + &optval, + sizeof(optval)); + if (rc) { + perror("setsockopt"); + } + struct sockaddr_in addr; memset(&addr, 0, sizeof(addr)); addr.sin_family = AF_INET; @@ -324,6 +346,28 @@ static int tcp_connect(xni_context_t ctx_, struct xni_endpoint* remote, xni_conn #endif // HAVE_DECL_TCP_CONGESTION } + //TODO: replace with a real value + int optval = 131072; + int rc = setsockopt(servers[i].sockd, + SOL_SOCKET, + SO_SNDBUF, + &optval, + sizeof(optval)); + if (rc) { + perror("setsockopt"); + } + + //TODO: replace with a real value + optval = 131072; + rc = setsockopt(servers[i].sockd, + SOL_SOCKET, + SO_RCVBUF, + &optval, + sizeof(optval)); + if (rc) { + perror("setsockopt"); + } + struct sockaddr_in addr; memset(&addr, 0, sizeof(addr)); addr.sin_family = AF_INET; From df57f9691f5fc03a9a1f8ef7e2abeba7fa1e44a2 Mon Sep 17 00:00:00 2001 From: "Bradley W. Settlemyer" Date: Mon, 12 May 2014 12:28:39 -0400 Subject: [PATCH 13/50] Revert "Add required Python packages" This reverts commit a666d802f63a36a5deb17b9a432920b7392bf29c. --- contrib/site-packages/Pyro4/__init__.py | 65 - contrib/site-packages/Pyro4/__init__.pyo | Bin 2660 -> 0 bytes contrib/site-packages/Pyro4/configuration.py | 128 - contrib/site-packages/Pyro4/configuration.pyo | Bin 5684 -> 0 bytes contrib/site-packages/Pyro4/constants.py | 20 - contrib/site-packages/Pyro4/constants.pyo | Bin 533 -> 0 bytes contrib/site-packages/Pyro4/core.py | 934 ------- contrib/site-packages/Pyro4/core.pyo | Bin 45729 -> 0 bytes contrib/site-packages/Pyro4/errors.py | 48 - contrib/site-packages/Pyro4/errors.pyo | Bin 2923 -> 0 bytes contrib/site-packages/Pyro4/futures.py | 147 -- contrib/site-packages/Pyro4/futures.pyo | Bin 7569 -> 0 bytes contrib/site-packages/Pyro4/message.py | 193 -- contrib/site-packages/Pyro4/message.pyo | Bin 8857 -> 0 bytes contrib/site-packages/Pyro4/naming.py | 384 --- contrib/site-packages/Pyro4/naming.pyo | Bin 16892 -> 0 bytes contrib/site-packages/Pyro4/nsc.py | 102 - contrib/site-packages/Pyro4/nsc.pyo | Bin 5439 -> 0 bytes .../Pyro4/socketserver/__init__.py | 5 - .../Pyro4/socketserver/__init__.pyo | Bin 349 -> 0 bytes .../Pyro4/socketserver/multiplexserver.py | 223 -- .../Pyro4/socketserver/multiplexserver.pyo | Bin 9764 -> 0 bytes .../Pyro4/socketserver/threadpoolserver.py | 188 -- .../Pyro4/socketserver/threadpoolserver.pyo | Bin 9070 -> 0 bytes contrib/site-packages/Pyro4/socketutil.py | 520 ---- contrib/site-packages/Pyro4/socketutil.pyo | Bin 18933 -> 0 bytes contrib/site-packages/Pyro4/test/__init__.py | 1 - contrib/site-packages/Pyro4/test/__init__.pyo | Bin 197 -> 0 bytes .../site-packages/Pyro4/test/echoserver.py | 126 - .../site-packages/Pyro4/test/echoserver.pyo | Bin 6168 -> 0 bytes contrib/site-packages/Pyro4/threadutil.py | 19 - contrib/site-packages/Pyro4/threadutil.pyo | Bin 801 -> 0 bytes contrib/site-packages/Pyro4/tpjobqueue.py | 203 -- contrib/site-packages/Pyro4/tpjobqueue.pyo | Bin 10078 -> 0 bytes contrib/site-packages/Pyro4/util.py | 546 ----- contrib/site-packages/Pyro4/util.pyo | Bin 26310 -> 0 bytes contrib/site-packages/Pyro4/utils/__init__.py | 1 - .../site-packages/Pyro4/utils/__init__.pyo | Bin 198 -> 0 bytes contrib/site-packages/Pyro4/utils/flame.py | 299 --- contrib/site-packages/Pyro4/utils/flame.pyo | Bin 17806 -> 0 bytes .../site-packages/Pyro4/utils/flameserver.py | 55 - .../site-packages/Pyro4/utils/flameserver.pyo | Bin 2471 -> 0 bytes contrib/site-packages/paramiko/__init__.py | 143 -- contrib/site-packages/paramiko/_winapi.py | 269 -- contrib/site-packages/paramiko/agent.py | 380 --- .../site-packages/paramiko/auth_handler.py | 426 ---- contrib/site-packages/paramiko/ber.py | 129 - .../site-packages/paramiko/buffered_pipe.py | 200 -- contrib/site-packages/paramiko/channel.py | 1279 ---------- contrib/site-packages/paramiko/client.py | 538 ---- contrib/site-packages/paramiko/common.py | 129 - contrib/site-packages/paramiko/compress.py | 39 - contrib/site-packages/paramiko/config.py | 266 -- contrib/site-packages/paramiko/dsskey.py | 196 -- contrib/site-packages/paramiko/ecdsakey.py | 181 -- contrib/site-packages/paramiko/file.py | 460 ---- contrib/site-packages/paramiko/hostkeys.py | 342 --- contrib/site-packages/paramiko/kex_gex.py | 243 -- contrib/site-packages/paramiko/kex_group1.py | 135 -- contrib/site-packages/paramiko/logging22.py | 66 - contrib/site-packages/paramiko/message.py | 302 --- contrib/site-packages/paramiko/packet.py | 500 ---- contrib/site-packages/paramiko/pipe.py | 147 -- contrib/site-packages/paramiko/pkey.py | 381 --- contrib/site-packages/paramiko/primes.py | 151 -- contrib/site-packages/paramiko/proxy.py | 91 - contrib/site-packages/paramiko/resource.py | 72 - contrib/site-packages/paramiko/rsakey.py | 185 -- contrib/site-packages/paramiko/server.py | 669 ----- contrib/site-packages/paramiko/sftp.py | 188 -- contrib/site-packages/paramiko/sftp_attr.py | 223 -- contrib/site-packages/paramiko/sftp_client.py | 787 ------ contrib/site-packages/paramiko/sftp_file.py | 489 ---- contrib/site-packages/paramiko/sftp_handle.py | 202 -- contrib/site-packages/paramiko/sftp_server.py | 444 ---- contrib/site-packages/paramiko/sftp_si.py | 310 --- .../site-packages/paramiko/ssh_exception.py | 132 - contrib/site-packages/paramiko/transport.py | 2158 ----------------- contrib/site-packages/paramiko/util.py | 311 --- contrib/site-packages/paramiko/win_pageant.py | 125 - 80 files changed, 17495 deletions(-) delete mode 100644 contrib/site-packages/Pyro4/__init__.py delete mode 100644 contrib/site-packages/Pyro4/__init__.pyo delete mode 100644 contrib/site-packages/Pyro4/configuration.py delete mode 100644 contrib/site-packages/Pyro4/configuration.pyo delete mode 100644 contrib/site-packages/Pyro4/constants.py delete mode 100644 contrib/site-packages/Pyro4/constants.pyo delete mode 100644 contrib/site-packages/Pyro4/core.py delete mode 100644 contrib/site-packages/Pyro4/core.pyo delete mode 100644 contrib/site-packages/Pyro4/errors.py delete mode 100644 contrib/site-packages/Pyro4/errors.pyo delete mode 100644 contrib/site-packages/Pyro4/futures.py delete mode 100644 contrib/site-packages/Pyro4/futures.pyo delete mode 100644 contrib/site-packages/Pyro4/message.py delete mode 100644 contrib/site-packages/Pyro4/message.pyo delete mode 100644 contrib/site-packages/Pyro4/naming.py delete mode 100644 contrib/site-packages/Pyro4/naming.pyo delete mode 100644 contrib/site-packages/Pyro4/nsc.py delete mode 100644 contrib/site-packages/Pyro4/nsc.pyo delete mode 100644 contrib/site-packages/Pyro4/socketserver/__init__.py delete mode 100644 contrib/site-packages/Pyro4/socketserver/__init__.pyo delete mode 100644 contrib/site-packages/Pyro4/socketserver/multiplexserver.py delete mode 100644 contrib/site-packages/Pyro4/socketserver/multiplexserver.pyo delete mode 100644 contrib/site-packages/Pyro4/socketserver/threadpoolserver.py delete mode 100644 contrib/site-packages/Pyro4/socketserver/threadpoolserver.pyo delete mode 100644 contrib/site-packages/Pyro4/socketutil.py delete mode 100644 contrib/site-packages/Pyro4/socketutil.pyo delete mode 100644 contrib/site-packages/Pyro4/test/__init__.py delete mode 100644 contrib/site-packages/Pyro4/test/__init__.pyo delete mode 100644 contrib/site-packages/Pyro4/test/echoserver.py delete mode 100644 contrib/site-packages/Pyro4/test/echoserver.pyo delete mode 100644 contrib/site-packages/Pyro4/threadutil.py delete mode 100644 contrib/site-packages/Pyro4/threadutil.pyo delete mode 100644 contrib/site-packages/Pyro4/tpjobqueue.py delete mode 100644 contrib/site-packages/Pyro4/tpjobqueue.pyo delete mode 100644 contrib/site-packages/Pyro4/util.py delete mode 100644 contrib/site-packages/Pyro4/util.pyo delete mode 100644 contrib/site-packages/Pyro4/utils/__init__.py delete mode 100644 contrib/site-packages/Pyro4/utils/__init__.pyo delete mode 100644 contrib/site-packages/Pyro4/utils/flame.py delete mode 100644 contrib/site-packages/Pyro4/utils/flame.pyo delete mode 100644 contrib/site-packages/Pyro4/utils/flameserver.py delete mode 100644 contrib/site-packages/Pyro4/utils/flameserver.pyo delete mode 100644 contrib/site-packages/paramiko/__init__.py delete mode 100644 contrib/site-packages/paramiko/_winapi.py delete mode 100644 contrib/site-packages/paramiko/agent.py delete mode 100644 contrib/site-packages/paramiko/auth_handler.py delete mode 100644 contrib/site-packages/paramiko/ber.py delete mode 100644 contrib/site-packages/paramiko/buffered_pipe.py delete mode 100644 contrib/site-packages/paramiko/channel.py delete mode 100644 contrib/site-packages/paramiko/client.py delete mode 100644 contrib/site-packages/paramiko/common.py delete mode 100644 contrib/site-packages/paramiko/compress.py delete mode 100644 contrib/site-packages/paramiko/config.py delete mode 100644 contrib/site-packages/paramiko/dsskey.py delete mode 100644 contrib/site-packages/paramiko/ecdsakey.py delete mode 100644 contrib/site-packages/paramiko/file.py delete mode 100644 contrib/site-packages/paramiko/hostkeys.py delete mode 100644 contrib/site-packages/paramiko/kex_gex.py delete mode 100644 contrib/site-packages/paramiko/kex_group1.py delete mode 100644 contrib/site-packages/paramiko/logging22.py delete mode 100644 contrib/site-packages/paramiko/message.py delete mode 100644 contrib/site-packages/paramiko/packet.py delete mode 100644 contrib/site-packages/paramiko/pipe.py delete mode 100644 contrib/site-packages/paramiko/pkey.py delete mode 100644 contrib/site-packages/paramiko/primes.py delete mode 100644 contrib/site-packages/paramiko/proxy.py delete mode 100644 contrib/site-packages/paramiko/resource.py delete mode 100644 contrib/site-packages/paramiko/rsakey.py delete mode 100644 contrib/site-packages/paramiko/server.py delete mode 100644 contrib/site-packages/paramiko/sftp.py delete mode 100644 contrib/site-packages/paramiko/sftp_attr.py delete mode 100644 contrib/site-packages/paramiko/sftp_client.py delete mode 100644 contrib/site-packages/paramiko/sftp_file.py delete mode 100644 contrib/site-packages/paramiko/sftp_handle.py delete mode 100644 contrib/site-packages/paramiko/sftp_server.py delete mode 100644 contrib/site-packages/paramiko/sftp_si.py delete mode 100644 contrib/site-packages/paramiko/ssh_exception.py delete mode 100644 contrib/site-packages/paramiko/transport.py delete mode 100644 contrib/site-packages/paramiko/util.py delete mode 100644 contrib/site-packages/paramiko/win_pageant.py diff --git a/contrib/site-packages/Pyro4/__init__.py b/contrib/site-packages/Pyro4/__init__.py deleted file mode 100644 index 4dca7320..00000000 --- a/contrib/site-packages/Pyro4/__init__.py +++ /dev/null @@ -1,65 +0,0 @@ -""" -Pyro package. Some generic init stuff to set up logging etc. - -Pyro - Python Remote Objects. Copyright by Irmen de Jong (irmen@razorvine.net). -""" - -import sys - -if sys.version_info < (2, 6): - import warnings - warnings.warn("This Pyro version is unsupported on Python versions older than 2.6", ImportWarning) - - -def _configLogging(): - """Do some basic config of the logging module at package import time. - The configuring is done only if the PYRO_LOGLEVEL env var is set. - If you want to use your own logging config, make sure you do - that before any Pyro imports. Then Pyro will skip the autoconfig. - Set the env var PYRO_LOGFILE to change the name of the autoconfigured - log file (default is pyro.log in the current dir). Use '{stderr}' to - make the log go to the standard error output.""" - import os, logging - level = os.environ.get("PYRO_LOGLEVEL") - logfilename = os.environ.get("PYRO_LOGFILE", "pyro.log") - if logfilename == "{stderr}": - logfilename = None - if level is not None: - levelvalue = getattr(logging, level) - if len(logging.root.handlers) == 0: - # configure the logging with some sensible defaults. - try: - import tempfile - tempfile = tempfile.TemporaryFile(dir=".") - tempfile.close() - except OSError: - # cannot write in current directory, use the default console logger - logging.basicConfig(level=levelvalue) - else: - # set up a basic logfile in current directory - logging.basicConfig( - level=levelvalue, - filename=logfilename, - datefmt="%Y-%m-%d %H:%M:%S", - format="[%(asctime)s.%(msecs)03d,%(name)s,%(levelname)s] %(message)s" - ) - log = logging.getLogger("Pyro4") - log.info("Pyro log configured using built-in defaults, level=%s", level) - else: - # PYRO_LOGLEVEL is not set, disable Pyro logging. No message is printed about this fact. - log = logging.getLogger("Pyro4") - log.setLevel(9999) - -_configLogging() -del _configLogging - -# initialize Pyro's configuration -from Pyro4.configuration import Configuration -config=Configuration() -del Configuration - -# import the required Pyro symbols into this package -from Pyro4.core import URI, Proxy, Daemon, callback, batch, async -from Pyro4.naming import locateNS, resolve -from Pyro4.futures import Future -from Pyro4.constants import VERSION as __version__ diff --git a/contrib/site-packages/Pyro4/__init__.pyo b/contrib/site-packages/Pyro4/__init__.pyo deleted file mode 100644 index b38df93fbd30e0631ad3bb0d04060440559061ad..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2660 zcmc&#{camY5TErWj_rInP1;hVQnnNmM@r79d`VHMD%u2sLQ-AQmbwD#>{~nM+})bp zy#zz@S9t(lfp;3P9<)7B9t>w7ngulruS*~@Tv>)+Fhm(t8D0a3 zS7lx?2gw|y6=+vLs0vxyRd`ilTLqFDr1Q|8hjanj3y>~CdlAwlXfHuphjty(WoR#h zAXrsMu0gs2?G;E@p}ksYYmi)r^aiwV6!JVIHz9os+P6?H!C`Ze68sg1+S9SoauoJX z!h!PTp-zp zbcd?vtS{wbJsO*MaO`AvEO$++G8rlPwZ`?$n8ZgW{7IYBI8%P6+`3$zTDe>zP%?^b}COpL!L;_ zBFb)Gj&&~2!VF_;nOjA&k@_sV1XJv?A=B_gNt+vX!Oa*3at8UT`Wn??HZF))fJIIt zNFp0@@hnafX;0!2=NaZs7hww+hj?5l|0%>N@SWYgZNk+<{ zc;aL~#@aO_)erN;QM!1pv`>~e;~2f%7=;-}v03-!GfaBzXY23?n2R;sjstPFlb*y*1u?)>5)I=U#P(7EPFzAFtSV=Jhb!eL>$af}9V9uA1gAy2@^ z69JH@Q`2# z(DsNs>(*~HQ>%J*{lkxUUKV}K~M`nwx3`qZYGs-%00YL za!Um4=5gXSvBHzW+YQMvKX2H$z6Q`FsE+O`Z8?R><;Zhvd~9^a`45zHGw8ODbY>$e zLgx${u>@>1T05=^G9yaxqGQ3SbVOz1P8^}2O=!kDq@h~&5{=BK@*Ny*Q~%sN&iNod z=5y%i#>4IQ@ZMI2H;S{q=GeG$k9yIh^#M5sH5v#i$H9CqTSOMl{xnQ-6|9gC^%i}a z_y;^#>Um3#Tr1JNFlp_@U4q-%G1!zdZBA^fxTyX9RMUE1-08M;-~X(Y#NF12UlWpS z?9}GuaiW#jvR0=`<(#0o~ zJ-VUm9rG--F zqLIDRiFB{iVQ_YA`B+XKKP=Fs!Dpe#uqc+>P6DU6%NOs9!WO(k5$P)D^7_+5!1)Jv z$Vl)CwIdQt{fr9UMKO>x+@;|?8t&2XJ`TlAK|S;F5F(7AL8cGL#McU&`kc27)<|8) zVH!T8d&${=c*SJwbf)=rt_b&ku&#oL9u)7{Lpp=@HV);|0&;0Vta{b8IpKNB$UD#T J#Z9p+{smq5Sc8e$7XM5n}7DxuS2D=pl#@FqkX+zIiW5Fdot z+?o_#M!1t=FD1e^!keVl&J>l;p7$|rN;L8rT51nPt{C*&{?0ITv>)_Mjl~~Yug>KrPYdQ7K6bk^mlf(xjiz=VbANEu4jG|^moj>Pu0hv z^V=Xi@cZ6M-_y5Oa{j;ZSJg9E^`k{wXyCugD4*b~9-{D|1BmqnOMIDxgSe2@y&}GY zN+yMSl^Y@IH72HnYcg?4$VUD;rqB~8p4S>pJq4oLu*d0oO*VBlYJXDvxW@fXgIBDr z*QM2HlxtO;L%m_iSC(vU)-6r!QmxsvO4Y);)!;h1uQ$u-ActlGJH<^SQJN zrE>MzJw1zRt!ix)Htnd@D(TN?N~>8YSFDoVl!c=8q)>bjEnF+Uu%-2~Vd=Bzg-t!R zq+PDt*beXR{L%FFTCHNQm#fjV4SkYRc%5+OKOR#qRjg#Ur@(r?UMSixtWDNZ;bpT{ zm$lcMyz?irRwxwLcuk}YB12N-Js=kS1bN@=!B!yr=ipH-Ui{GT8&&` zrJb~_zVbo^uTiV_R=!^C`rE4mb{48BUwd~3PHWHE@n~cie6`vdceNGtB`w>w-}kj` zUjnUCD1_0XA2M0;QkLxrS^xnM&rXCSnn8gGfI^zNK!gP*;2#kI|Cr#GDS`EOj`x|jP*F@>fU&SFLPl@+waY37+ z^M%;Ui156juTvMaen^SK2@$d4ehY_?A4sww_Oc@U-q1Iw`oR!~&_YJ&n_};j2>)!n zpA?;2gqzeK3cbX0|IUCO(X#&+>fkPN;~x!ZCu-C`5$=4vC3I+~#XP3|))4#O8<-tU zw9=%2z3HWRlhl|N#{PNw-vEcin-}4Y6zz>^4AObt@QqZ|(zz|%3p5$3_B_w`Q|^34 zbbcn>i^Ph#S4h}K{xa-I{T@ZmjQ<**25h&TVON`xd73cX9`HZuQEr<<<$3)BKMeYS zb#3~}eCl+Sw`{%*`b*jjJ;yc2I~+Kn?`(HHbrep_JmzQ*yQbeZN5Rl+IW$m*L#3%6 z(r`qGWKBjKcEjC3Y5&_BLiiu3z}AH(uh6@+>JHWEFCiZLNLur-PYuq{eanRQUbHT zWcJ8Y%*JTi1*G%TRS!(@?kX{DP-iuI6d5 zyoo}^i!vQfN`kth9C4~Ta0R>CM z$0|v`lHf-YK1qs^lA@O+gq0NcSiUCYlhf2h5nU40ONvdB!k9cq1s#FBNCgEUxj+R4 zK1pCF7pWkV}^;%&V zgz-s7!${oik;WO3d|tm5xE>!Hr98w`gRZahEH|B+CTTymd>%=mxBQw(HR^50%k!Bs zo@2{Vh4dCt*er(_E#h<7&`ppfRgSs);X$pgFq%Rp%WJczaZIk5pU z3f_u5IrQ`}>?>36dS+`FhJ-;eRyoRafImCKt`nNB-=Z8AK|9Yje-~rr&r#&r`uR}l z?2!fl?$~??>ptnxupK$7gss2E))aZ1U^QQe(g0E-ZJ!k>K>+F~0$QXmao!Lk2A zr0@a2-;Gd|&4PUONKi?-X7iw?Pir28pirao3du1v_!N)^Mj5Y|%oy|Rjahgl`ySa4 z!5Ev&J$Pe$c@(0D!~+LUbs@S)jNpN35)}pl4-D@U-Qf!(*~=ouTHG9wK+;{ZMtNf& zW|d+4LRNB$Z4KEKx@AQtN9i9di*l<|V(qv!o3!S{8l`xKm`8#~{L`XyM9i>xBH^5+ z?)XQtw}78{_Y7GxZEcPgB;qU+yu0=Udd!NQ2}-KoeJyZ}!Uj6$s0T)u-rNwsf~eRu zQd?wnNK%o=A`zPs?G)!`#{R#=hI>|Y&d1qVBTq4`$`{t?zCijMziLH^zU}u0U5}C? zzCB}OP>OTlKr$h|;ky6_=tqwpX^NDbP~Zad6gNB$P6<4B93^b` z6-kUC571u`qUZ9IEAZ`)Z!;?xsI-F|N>+$L{ecGqJP$Cv>nIM_WR37%-44Q@Tt`Jx zIAB%vakG*ZH4)5nFh-DP*1*xbJdLxhh?eZdv<7;#{jNt)7{%!LC}L4r^=hO5>=B7H zz;oIEIkA<`X&fI^;P>;iJ4p&-A>_+s(h~L; ze5xWf2Q-*9W>bs!oyTX&ST?RBN?$V;jpznIgXpTxXB*!B9Ts`XOV`hr$|__ z3qeX!bWzA4WQ~6^scd@sm)WW8Oe|;*iHL2pFEeJ?wi~n{Dg#ZLrWq-@fnP?1fS)iK fKjaP?|4(FQS=ORsddT+?iWr#;C)(-xVA}W(-X;>^ diff --git a/contrib/site-packages/Pyro4/constants.py b/contrib/site-packages/Pyro4/constants.py deleted file mode 100644 index 546d39e7..00000000 --- a/contrib/site-packages/Pyro4/constants.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -Definitions of various hard coded constants. - -Pyro - Python Remote Objects. Copyright by Irmen de Jong (irmen@razorvine.net). -""" - -# Pyro version -VERSION = "4.22" - -# standard object name for the Daemon object -DAEMON_NAME = "Pyro.Daemon" - -# standard name for the Name server itself -NAMESERVER_NAME = "Pyro.NameServer" - -# standard name for Flame server -FLAME_NAME = "Pyro.Flame" - -# wire protocol version -PROTOCOL_VERSION = 46 diff --git a/contrib/site-packages/Pyro4/constants.pyo b/contrib/site-packages/Pyro4/constants.pyo deleted file mode 100644 index 544089b24c05e4ab04b466f1dbdfbb0c55579eb7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 533 zcmYLFOK!q26g1yfsjBt}UZx9T2_bb;RUkeR6(lHBEV5{1VgrU?Bim4=t6rsN>k)c^ zKJyWdJb5#Ip1e%u@80|P{fRSlUL|<9VH+QSA(Q}?P*OsyjFK{96_ivEtD>ZeSPdmL z#Oi43-?$s!0A7zTgcMdNX|P)01Jy$94PH^raHbe{D2=7kn!xiWg;uzMCxu-piDSN1 zmg8u?<{2Dg+*P|mi{;AVd4Y#|%Oz$UzbU!IzHqdy>6g+6A$cIVeF{8Ng&|r&vuW<2 zaP~ouf~_= (3, 0): - basestring = str - -log = logging.getLogger("Pyro4.core") - - -class URI(object): - """ - Pyro object URI (universal resource identifier). - The uri format is like this: ``PYRO:objectid@location`` where location is one of: - - - ``hostname:port`` (tcp/ip socket on given port) - - ``./u:sockname`` (Unix domain socket on localhost) - - There is also a 'Magic format' for simple name resolution using Name server: - ``PYRONAME:objectname[@location]`` (optional name server location, can also omit location port) - - You can write the protocol in lowercase if you like (``pyro:...``) but it will - automatically be converted to uppercase internally. - """ - uriRegEx = re.compile(r"(?P[Pp][Yy][Rr][Oo][a-zA-Z]*):(?P\S+?)(@(?P\S+))?$") - __slots__ = ("protocol", "object", "sockname", "host", "port", "object") - - def __init__(self, uri): - if isinstance(uri, URI): - state=uri.__getstate__() - self.__setstate__(state) - return - if not isinstance(uri, basestring): - raise TypeError("uri parameter object is of wrong type") - self.sockname=self.host=self.port=None - match=self.uriRegEx.match(uri) - if not match: - raise errors.PyroError("invalid uri") - self.protocol=match.group("protocol").upper() - self.object=match.group("object") - location=match.group("location") - if self.protocol=="PYRONAME": - self._parseLocation(location, Pyro4.config.NS_PORT) - return - if self.protocol=="PYRO": - if not location: - raise errors.PyroError("invalid uri") - self._parseLocation(location, None) - else: - raise errors.PyroError("invalid uri (protocol)") - - def _parseLocation(self, location, defaultPort): - if not location: - return - if location.startswith("./u:"): - self.sockname=location[4:] - if (not self.sockname) or ':' in self.sockname: - raise errors.PyroError("invalid uri (location)") - else: - if location.startswith("["): # ipv6 - if location.startswith("[["): # possible mistake: double-bracketing - raise errors.PyroError("invalid ipv6 address: enclosed in too many brackets") - self.host, _, self.port = re.match(r"\[([0-9a-fA-F:%]+)](:(\d+))?", location).groups() - else: - self.host, _, self.port = location.partition(":") - if not self.port: - self.port=defaultPort - try: - self.port=int(self.port) - except (ValueError, TypeError): - raise errors.PyroError("invalid port in uri, port="+str(self.port)) - - @staticmethod - def isUnixsockLocation(location): - """determine if a location string is for a Unix domain socket""" - return location.startswith("./u:") - - @property - def location(self): - """property containing the location string, for instance ``"servername.you.com:5555"``""" - if self.host: - if ":" in self.host: # ipv6 - return "[%s]:%d" % (self.host, self.port) - else: - return "%s:%d" % (self.host, self.port) - elif self.sockname: - return "./u:"+self.sockname - else: - return None - - def asString(self): - """the string representation of this object""" - result=self.protocol+":"+self.object - location=self.location - if location: - result+="@"+location - return result - - def __str__(self): - string=self.asString() - if sys.version_info<(3, 0) and type(string) is unicode: - return string.encode("ascii", "replace") - return string - - def __unicode__(self): - return self.asString() - - def __repr__(self): - return "<%s.%s at 0x%x, %s>" % (self.__class__.__module__, self.__class__.__name__, id(self), str(self)) - - def __eq__(self, other): - if not isinstance(other, URI): - return False - return (self.protocol, self.object, self.sockname, self.host, self.port) \ - == (other.protocol, other.object, other.sockname, other.host, other.port) - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash((self.protocol, self.object, self.sockname, self.host, self.port)) - - # note: getstate/setstate are not needed if we use pickle protocol 2, - # but this way it helps pickle to make the representation smaller by omitting all attribute names. - - def __getstate__(self): - return self.protocol, self.object, self.sockname, self.host, self.port - - def __getstate_for_dict__(self): - return self.__getstate__() - - def __setstate__(self, state): - self.protocol, self.object, self.sockname, self.host, self.port = state - - -class _RemoteMethod(object): - """method call abstraction""" - def __init__(self, send, name): - self.__send = send - self.__name = name - - def __getattr__(self, name): - return _RemoteMethod(self.__send, "%s.%s" % (self.__name, name)) - - def __call__(self, *args, **kwargs): - return self.__send(self.__name, args, kwargs) - - -def _check_hmac(): - if Pyro4.config.HMAC_KEY: - if sys.version_info>=(3, 0) and type(Pyro4.config.HMAC_KEY) is not bytes: - raise errors.PyroError("HMAC_KEY must be bytes type") - - -class Proxy(object): - """ - Pyro proxy for a remote object. Intercepts method calls and dispatches them to the remote object. - - .. automethod:: _pyroBind - .. automethod:: _pyroRelease - .. automethod:: _pyroReconnect - .. automethod:: _pyroBatch - .. automethod:: _pyroAsync - """ - __pyroAttributes=frozenset(["__getnewargs__", "__getinitargs__", "_pyroConnection", "_pyroUri", "_pyroOneway", "_pyroTimeout", "_pyroSeq"]) - - def __init__(self, uri): - """ - .. autoattribute:: _pyroOneway - .. autoattribute:: _pyroTimeout - """ - _check_hmac() # check if hmac secret key is set - if isinstance(uri, basestring): - uri=URI(uri) - elif not isinstance(uri, URI): - raise TypeError("expected Pyro URI") - self._pyroUri=uri - self._pyroConnection=None - self._pyroOneway=set() - self._pyroSeq=0 # message sequence number - self.__pyroTimeout=Pyro4.config.COMMTIMEOUT - self.__pyroLock=threadutil.Lock() - self.__pyroConnLock=threadutil.Lock() - util.get_serializer(Pyro4.config.SERIALIZER) # assert that the configured serializer is available - if os.name=="java" and Pyro4.config.SERIALIZER=="marshal": - import warnings - warnings.warn("marshal doesn't work correctly with Jython (issue 2077); please choose another serializer", RuntimeWarning) - - def __del__(self): - if hasattr(self, "_pyroConnection"): - self._pyroRelease() - - def __getattr__(self, name): - if name in Proxy.__pyroAttributes: - # allows it to be safely pickled - raise AttributeError(name) - return _RemoteMethod(self._pyroInvoke, name) - - def __repr__(self): - connected="connected" if self._pyroConnection else "not connected" - return "<%s.%s at 0x%x, %s, for %s>" % (self.__class__.__module__, self.__class__.__name__, - id(self), connected, self._pyroUri) - - def __unicode__(self): - return str(self) - - def __getstate__(self): - return self._pyroUri, self._pyroOneway, self.__pyroTimeout # skip the connection - - def __getstate_for_dict__(self): - return self._pyroUri.asString(), tuple(self._pyroOneway), self.__pyroTimeout - - def __setstate__(self, state): - self._pyroUri, self._pyroOneway, self.__pyroTimeout = state - self._pyroConnection=None - self._pyroSeq=0 - self.__pyroLock=threadutil.Lock() - self.__pyroConnLock=threadutil.Lock() - - def __copy__(self): - uriCopy=URI(self._pyroUri) - return Proxy(uriCopy) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self._pyroRelease() - - def __eq__(self, other): - if other is self: - return True - return isinstance(other, Proxy) and other._pyroUri == self._pyroUri and other._pyroOneway == self._pyroOneway - - def __ne__(self, other): - if other and isinstance(other, Proxy): - return other._pyroUri != self._pyroUri or other._pyroOneway != self._pyroOneway - return True - - def __hash__(self): - return hash(self._pyroUri) ^ hash(frozenset(self._pyroOneway)) - - def _pyroRelease(self): - """release the connection to the pyro daemon""" - with self.__pyroConnLock: - if self._pyroConnection is not None: - self._pyroConnection.close() - self._pyroConnection=None - log.debug("connection released") - - def _pyroBind(self): - """ - Bind this proxy to the exact object from the uri. That means that the proxy's uri - will be updated with a direct PYRO uri, if it isn't one yet. - If the proxy is already bound, it will not bind again. - """ - return self.__pyroCreateConnection(True) - - def __pyroGetTimeout(self): - return self.__pyroTimeout - - def __pyroSetTimeout(self, timeout): - self.__pyroTimeout=timeout - if self._pyroConnection is not None: - self._pyroConnection.timeout=timeout - _pyroTimeout=property(__pyroGetTimeout, __pyroSetTimeout) - - def _pyroInvoke(self, methodname, vargs, kwargs, flags=0): - """perform the remote method call communication""" - if self._pyroConnection is None: - # rebind here, don't do it from inside the invoke because deadlock will occur - self.__pyroCreateConnection() - serializer = util.get_serializer(Pyro4.config.SERIALIZER) - data, compressed = serializer.serializeCall( - self._pyroConnection.objectId, methodname, vargs, kwargs, - compress=Pyro4.config.COMPRESSION) - if compressed: - flags |= Pyro4.message.FLAGS_COMPRESSED - if methodname in self._pyroOneway: - flags |= Pyro4.message.FLAGS_ONEWAY - with self.__pyroLock: - self._pyroSeq=(self._pyroSeq+1)&0xffff - if Pyro4.config.LOGWIRE: - log.debug("proxy wiredata sending: msgtype=%d flags=0x%x ser=%d seq=%d data=%r" % (Pyro4.message.MSG_INVOKE, flags, serializer.serializer_id, self._pyroSeq, data)) - msg = Message(Pyro4.message.MSG_INVOKE, data, serializer.serializer_id, flags, self._pyroSeq) - try: - self._pyroConnection.send(msg.to_bytes()) - del msg # invite GC to collect the object, don't wait for out-of-scope - if flags & Pyro4.message.FLAGS_ONEWAY: - return None # oneway call, no response data - else: - msg = Message.recv(self._pyroConnection, [Pyro4.message.MSG_RESULT]) - if Pyro4.config.LOGWIRE: - log.debug("proxy wiredata received: msgtype=%d flags=0x%x ser=%d seq=%d data=%r" % (msg.type, msg.flags, msg.serializer_id, msg.seq, msg.data) ) - self.__pyroCheckSequence(msg.seq) - if msg.serializer_id != serializer.serializer_id: - error = "invalid serializer in response: %d" % msg.serializer_id - log.error(error) - raise errors.ProtocolError(error) - data = serializer.deserializeData(msg.data, compressed=msg.flags & Pyro4.message.FLAGS_COMPRESSED) - if msg.flags & Pyro4.message.FLAGS_EXCEPTION: - if sys.platform=="cli": - util.fixIronPythonExceptionForPickle(data, False) - raise data - else: - return data - except (errors.CommunicationError, KeyboardInterrupt): - # Communication error during read. To avoid corrupt transfers, we close the connection. - # Otherwise we might receive the previous reply as a result of a new methodcall! - # Special case for keyboardinterrupt: people pressing ^C to abort the client - # may be catching the keyboardinterrupt in their code. We should probably be on the - # safe side and release the proxy connection in this case too, because they might - # be reusing the proxy object after catching the exception... - self._pyroRelease() - raise - - def __pyroCheckSequence(self, seq): - if seq!=self._pyroSeq: - err="invoke: reply sequence out of sync, got %d expected %d" % (seq, self._pyroSeq) - log.error(err) - raise errors.ProtocolError(err) - - def __pyroCreateConnection(self, replaceUri=False): - """ - Connects this proxy to the remote Pyro daemon. Does connection handshake. - Returns true if a new connection was made, false if an existing one was already present. - """ - with self.__pyroConnLock: - if self._pyroConnection is not None: - return False # already connected - from Pyro4.naming import resolve # don't import this globally because of cyclic dependancy - uri=resolve(self._pyroUri) - # socket connection (normal or Unix domain socket) - conn=None - log.debug("connecting to %s", uri) - connect_location=uri.sockname if uri.sockname else (uri.host, uri.port) - with self.__pyroLock: - try: - if self._pyroConnection is not None: - return False # already connected - sock=socketutil.createSocket(connect=connect_location, reuseaddr=Pyro4.config.SOCK_REUSE, timeout=self.__pyroTimeout) - conn=socketutil.SocketConnection(sock, uri.object) - # Do handshake. For now, no need to send anything. (message type CONNECT is not yet used) - msg = Message.recv(conn, None) - # any trailing data (dataLen>0) is an error message, if any - except Exception: - x=sys.exc_info()[1] - if conn: - conn.close() - err="cannot connect: %s" % x - log.error(err) - if isinstance(x, errors.CommunicationError): - raise - else: - ce = errors.CommunicationError(err) - ce.__cause__ = x - raise ce - else: - if msg.type==Pyro4.message.MSG_CONNECTFAIL: - error="connection rejected" - if msg.data: - data = msg.data - if sys.version_info>=(3, 0): - data=str(msg.data, "utf-8") - error+=", reason: " + data - conn.close() - log.error(error) - raise errors.CommunicationError(error) - elif msg.type==Pyro4.message.MSG_CONNECTOK: - self._pyroConnection=conn - if replaceUri: - self._pyroUri=uri - log.debug("connected to %s", self._pyroUri) - return True - else: - conn.close() - err="connect: invalid msg type %d received" % msg.type - log.error(err) - raise errors.ProtocolError(err) - - def _pyroReconnect(self, tries=100000000): - """(re)connect the proxy to the daemon containing the pyro object which the proxy is for""" - self._pyroRelease() - while tries: - try: - self.__pyroCreateConnection() - return - except errors.CommunicationError: - tries-=1 - if tries: - time.sleep(2) - msg="failed to reconnect" - log.error(msg) - raise errors.ConnectionClosedError(msg) - - def _pyroBatch(self): - """returns a helper class that lets you create batched method calls on the proxy""" - return _BatchProxyAdapter(self) - - def _pyroAsync(self): - """returns a helper class that lets you do asynchronous method calls on the proxy""" - return _AsyncProxyAdapter(self) - - def _pyroInvokeBatch(self, calls, oneway=False): - flags=Pyro4.message.FLAGS_BATCH - if oneway: - flags|=Pyro4.message.FLAGS_ONEWAY - return self._pyroInvoke("", calls, None, flags) - - -class _BatchedRemoteMethod(object): - """method call abstraction that is used with batched calls""" - def __init__(self, calls, name): - self.__calls = calls - self.__name = name - - def __getattr__(self, name): - return _BatchedRemoteMethod(self.__calls, "%s.%s" % (self.__name, name)) - - def __call__(self, *args, **kwargs): - self.__calls.append((self.__name, args, kwargs)) - - -class _BatchProxyAdapter(object): - """Helper class that lets you batch multiple method calls into one. - It is constructed with a reference to the normal proxy that will - carry out the batched calls. Call methods on this object thatyou want to batch, - and finally call the batch proxy itself. That call will return a generator - for the results of every method call in the batch (in sequence).""" - def __init__(self, proxy): - self.__proxy=proxy - self.__calls=[] - - def __getattr__(self, name): - return _BatchedRemoteMethod(self.__calls, name) - - def __enter__(self): - return self - - def __exit__(self, *args): - pass - - def __copy__(self): - return self - - def __resultsgenerator(self, results): - for result in results: - if isinstance(result, futures._ExceptionWrapper): - result.raiseIt() # re-raise the remote exception locally. - else: - yield result # it is a regular result object, yield that and continue. - - def __call__(self, oneway=False, async=False): - if oneway and async: - raise errors.PyroError("async oneway calls make no sense") - if async: - return _AsyncRemoteMethod(self, "")() - else: - results=self.__proxy._pyroInvokeBatch(self.__calls, oneway) - self.__calls=[] # clear for re-use - if not oneway: - return self.__resultsgenerator(results) - - def _pyroInvoke(self, name, args, kwargs): - # ignore all parameters, we just need to execute the batch - results=self.__proxy._pyroInvokeBatch(self.__calls) - self.__calls=[] # clear for re-use - return self.__resultsgenerator(results) - - -class _AsyncProxyAdapter(object): - def __init__(self, proxy): - self.__proxy=proxy - - def __getattr__(self, name): - return _AsyncRemoteMethod(self.__proxy, name) - - -class _AsyncRemoteMethod(object): - """async method call abstraction (call will run in a background thread)""" - def __init__(self, proxy, name): - self.__proxy = proxy - self.__name = name - - def __getattr__(self, name): - return _AsyncRemoteMethod(self.__proxy, "%s.%s" % (self.__name, name)) - - def __call__(self, *args, **kwargs): - result=futures.FutureResult() - thread=threadutil.Thread(target=self.__asynccall, args=(result, args, kwargs)) - thread.setDaemon(True) - thread.start() - return result - - def __asynccall(self, asyncresult, args, kwargs): - try: - # use a copy of the proxy otherwise calls would be serialized, - # and use contextmanager to close the proxy after we're done - with self.__proxy.__copy__() as proxy: - value = proxy._pyroInvoke(self.__name, args, kwargs) - asyncresult.value=value - except Exception: - # ignore any exceptions here, return them as part of the async result instead - asyncresult.value=futures._ExceptionWrapper(sys.exc_info()[1]) - - -def batch(proxy): - """convenience method to get a batch proxy adapter""" - return proxy._pyroBatch() - - -def async(proxy): - """convenience method to get an async proxy adapter""" - return proxy._pyroAsync() - - -def pyroObjectToAutoProxy(self): - """reduce function that automatically replaces Pyro objects by a Proxy""" - if Pyro4.config.AUTOPROXY: - daemon = getattr(self, "_pyroDaemon", None) - if daemon: - # only return a proxy if the object is a registered pyro object - return Pyro4.core.Proxy(daemon.uriFor(self)) - return self - - -class DaemonObject(object): - """The part of the daemon that is exposed as a Pyro object.""" - def __init__(self, daemon): - self.daemon=daemon - - def registered(self): - """returns a list of all object names registered in this daemon""" - return list(self.daemon.objectsById.keys()) - - def ping(self): - """a simple do-nothing method for testing purposes""" - pass - - -class Daemon(object): - """ - Pyro daemon. Contains server side logic and dispatches incoming remote method calls - to the appropriate objects. - """ - def __init__(self, host=None, port=0, unixsocket=None, nathost=None, natport=None): - _check_hmac() # check if hmac secret key is set - if host is None: - host=Pyro4.config.HOST - if nathost is None: - nathost=Pyro4.config.NATHOST - if natport is None: - natport=Pyro4.config.NATPORT or None - if nathost and unixsocket: - raise ValueError("cannot use nathost together with unixsocket") - if (nathost is None) ^ (natport is None): - raise ValueError("must provide natport with nathost") - if Pyro4.config.SERVERTYPE=="thread": - self.transportServer=SocketServer_Threadpool() - elif Pyro4.config.SERVERTYPE=="multiplex": - # choose the 'best' multiplexing implementation - if os.name=="java": - raise NotImplementedError("select or poll-based server is not supported for jython, use thread server instead") - self.transportServer = SocketServer_Poll() if socketutil.hasPoll else SocketServer_Select() - else: - raise errors.PyroError("invalid server type '%s'" % Pyro4.config.SERVERTYPE) - self.transportServer.init(self, host, port, unixsocket) - #: The location (str of the form ``host:portnumber``) on which the Daemon is listening - self.locationStr=self.transportServer.locationStr - log.debug("created daemon on %s", self.locationStr) - natport_for_loc = natport - if natport==0: - # expose internal port number as NAT port as well. (don't use port because it could be 0 and will be chosen by the OS) - natport_for_loc = int(self.locationStr.split(":")[1]) - #: The NAT-location (str of the form ``nathost:natportnumber``) on which the Daemon is exposed for use with NAT-routing - self.natLocationStr = "%s:%d" % (nathost, natport_for_loc) if nathost else None - if self.natLocationStr: - log.debug("NAT address is %s", self.natLocationStr) - pyroObject=DaemonObject(self) - pyroObject._pyroId=constants.DAEMON_NAME - #: Dictionary from Pyro object id to the actual Pyro object registered by this id - self.objectsById={pyroObject._pyroId: pyroObject} - self.__mustshutdown=threadutil.Event() - self.__loopstopped=threadutil.Event() - self.__loopstopped.set() - # assert that the configured serializers are available, and remember their ids: - self.__serializer_ids = set([util.get_serializer(ser_name).serializer_id for ser_name in Pyro4.config.SERIALIZERS_ACCEPTED]) - log.debug("accepted serializers: %s" % Pyro4.config.SERIALIZERS_ACCEPTED) - - @property - def sock(self): - return self.transportServer.sock - - @property - def sockets(self): - return self.transportServer.sockets - - @staticmethod - def serveSimple(objects, host=None, port=0, daemon=None, ns=True, verbose=True): - """ - Very basic method to fire up a daemon (or supply one yourself). - objects is a dict containing objects to register as keys, and - their names (or None) as values. If ns is true they will be registered - in the naming server as well, otherwise they just stay local. - """ - if not daemon: - daemon=Daemon(host, port) - with daemon: - if ns: - ns=Pyro4.naming.locateNS() - for obj, name in objects.items(): - if ns: - localname=None # name is used for the name server - else: - localname=name # no name server, use name in daemon - uri=daemon.register(obj, localname) - if verbose: - print("Object {0}:\n uri = {1}".format(repr(obj), uri)) - if name and ns: - ns.register(name, uri) - if verbose: - print(" name = {0}".format(name)) - if verbose: - print("Pyro daemon running.") - daemon.requestLoop() - - def requestLoop(self, loopCondition=lambda: True): - """ - Goes in a loop to service incoming requests, until someone breaks this - or calls shutdown from another thread. - """ - self.__mustshutdown.clear() - log.info("daemon %s entering requestloop", self.locationStr) - try: - self.__loopstopped.clear() - condition=lambda: not self.__mustshutdown.isSet() and loopCondition() - self.transportServer.loop(loopCondition=condition) - finally: - self.__loopstopped.set() - log.debug("daemon exits requestloop") - - def events(self, eventsockets): - """for use in an external event loop: handle any requests that are pending for this daemon""" - return self.transportServer.events(eventsockets) - - def shutdown(self): - """Cleanly terminate a daemon that is running in the requestloop. It must be running - in a different thread, or this method will deadlock.""" - log.debug("daemon shutting down") - self.__mustshutdown.set() - self.transportServer.wakeup() - time.sleep(0.05) - self.close() - self.__loopstopped.wait() - log.info("daemon %s shut down", self.locationStr) - - def _handshake(self, conn): - """Perform connection handshake with new clients""" - # For now, client is not sending anything. Just respond with a CONNECT_OK. - # We need a minimal amount of data or the socket will remain blocked - # on some systems... (messages smaller than 40 bytes) - # Return True for successful handshake, False if something was wrong. - # We default to the marshal serializer to send message payload of "ok" - ser = util.get_serializer("marshal") - data = ser.dumps("ok") - msg = Message(Pyro4.message.MSG_CONNECTOK, data, ser.serializer_id, 0, 1) - conn.send(msg.to_bytes()) - return True - - def handleRequest(self, conn): - """ - Handle incoming Pyro request. Catches any exception that may occur and - wraps it in a reply to the calling side, as to not make this server side loop - terminate due to exceptions caused by remote invocations. - """ - request_flags=0 - request_seq=0 - request_serializer_id = util.MarshalSerializer.serializer_id - wasBatched=False - isCallback=False - try: - msg = Message.recv(conn, [Pyro4.message.MSG_INVOKE, Pyro4.message.MSG_PING]) - request_flags = msg.flags - request_seq = msg.seq - request_serializer_id = msg.serializer_id - if Pyro4.config.LOGWIRE: - log.debug("daemon wiredata received: msgtype=%d flags=0x%x ser=%d seq=%d data=%r" % (msg.type, msg.flags, msg.serializer_id, msg.seq, msg.data) ) - if msg.type == Pyro4.message.MSG_PING: - # return same seq, but ignore any data (it's a ping, not an echo). Nothing is deserialized. - msg = Message(Pyro4.message.MSG_PING, b"pong", msg.serializer_id, 0, msg.seq) - if Pyro4.config.LOGWIRE: - log.debug("daemon wiredata sending: msgtype=%d flags=0x%x ser=%d seq=%d data=%r" % (msg.type, msg.flags, msg.serializer_id, msg.seq, msg.data)) - conn.send(msg.to_bytes()) - return - if msg.serializer_id not in self.__serializer_ids: - raise errors.ProtocolError("message used serializer that is not accepted: %d" % msg.serializer_id) - serializer = util.get_serializer_by_id(msg.serializer_id) - objId, method, vargs, kwargs = serializer.deserializeCall(msg.data, compressed=msg.flags & Pyro4.message.FLAGS_COMPRESSED) - del msg # invite GC to collect the object, don't wait for out-of-scope - obj = self.objectsById.get(objId) - if obj is not None: - if kwargs and sys.version_info<(2, 6, 5) and os.name!="java": - # Python before 2.6.5 doesn't accept unicode keyword arguments - kwargs = dict((str(k), kwargs[k]) for k in kwargs) - if request_flags & Pyro4.message.FLAGS_BATCH: - # batched method calls, loop over them all and collect all results - data=[] - for method, vargs, kwargs in vargs: - method=util.resolveDottedAttribute(obj, method, Pyro4.config.DOTTEDNAMES) - try: - result=method(*vargs, **kwargs) # this is the actual method call to the Pyro object - except Exception: - xt, xv = sys.exc_info()[0:2] - log.debug("Exception occurred while handling batched request: %s", xv) - xv._pyroTraceback=util.formatTraceback(detailed=Pyro4.config.DETAILED_TRACEBACK) - if sys.platform=="cli": - util.fixIronPythonExceptionForPickle(xv, True) # piggyback attributes - data.append(futures._ExceptionWrapper(xv)) - break # stop processing the rest of the batch - else: - data.append(result) - wasBatched=True - else: - # normal single method call - method=util.resolveDottedAttribute(obj, method, Pyro4.config.DOTTEDNAMES) - if request_flags & Pyro4.message.FLAGS_ONEWAY and Pyro4.config.ONEWAY_THREADED: - # oneway call to be run inside its own thread - thread=threadutil.Thread(target=method, args=vargs, kwargs=kwargs) - thread.setDaemon(True) - thread.start() - else: - isCallback=getattr(method, "_pyroCallback", False) - data=method(*vargs, **kwargs) # this is the actual method call to the Pyro object - else: - log.debug("unknown object requested: %s", objId) - raise errors.DaemonError("unknown object") - if request_flags & Pyro4.message.FLAGS_ONEWAY: - return # oneway call, don't send a response - else: - data, compressed = serializer.serializeData(data, compress=Pyro4.config.COMPRESSION) - response_flags=0 - if compressed: - response_flags |= Pyro4.message.FLAGS_COMPRESSED - if wasBatched: - response_flags |= Pyro4.message.FLAGS_BATCH - if Pyro4.config.LOGWIRE: - log.debug("daemon wiredata sending: msgtype=%d flags=0x%x ser=%d seq=%d data=%r" % (Pyro4.message.MSG_RESULT, response_flags, serializer.serializer_id, request_seq, data)) - msg = Message(Pyro4.message.MSG_RESULT, data, serializer.serializer_id, response_flags, request_seq) - conn.send(msg.to_bytes()) - except Exception: - xt, xv = sys.exc_info()[0:2] - if xt is not errors.ConnectionClosedError: - log.debug("Exception occurred while handling request: %r", xv) - if not request_flags & Pyro4.message.FLAGS_ONEWAY: - # only return the error to the client if it wasn't a oneway call - tblines=util.formatTraceback(detailed=Pyro4.config.DETAILED_TRACEBACK) - self._sendExceptionResponse(conn, request_seq, request_serializer_id, xv, tblines) - if isCallback or isinstance(xv, (errors.CommunicationError, errors.SecurityError)): - raise # re-raise if flagged as callback, communication or security error. - - def _sendExceptionResponse(self, connection, seq, serializer_id, exc_value, tbinfo): - """send an exception back including the local traceback info""" - exc_value._pyroTraceback=tbinfo - if sys.platform=="cli": - util.fixIronPythonExceptionForPickle(exc_value, True) # piggyback attributes - serializer = util.get_serializer_by_id(serializer_id) - try: - data, compressed = serializer.serializeData(exc_value) - except: - # the exception object couldn't be serialized, use a generic PyroError instead - xt, xv, tb = sys.exc_info() - msg = "Error serializing exception: %s. Original exception: %s: %s" % (str(xv), type(exc_value), str(exc_value)) - exc_value = errors.PyroError(msg) - exc_value._pyroTraceback=tbinfo - if sys.platform=="cli": - util.fixIronPythonExceptionForPickle(exc_value, True) # piggyback attributes - data, compressed = serializer.serializeData(exc_value) - flags = Pyro4.message.FLAGS_EXCEPTION - if compressed: - flags |= Pyro4.message.FLAGS_COMPRESSED - if Pyro4.config.LOGWIRE: - log.debug("daemon wiredata sending (error response): msgtype=%d flags=0x%x ser=%d seq=%d data=%r" % (Pyro4.message.MSG_RESULT, flags, serializer.serializer_id, seq, data)) - msg = Message(Pyro4.message.MSG_RESULT, data, serializer.serializer_id, flags, seq) - connection.send(msg.to_bytes()) - - def register(self, obj, objectId=None): - """ - Register a Pyro object under the given id. Note that this object is now only - known inside this daemon, it is not automatically available in a name server. - This method returns a URI for the registered object. - """ - if objectId: - if not isinstance(objectId, basestring): - raise TypeError("objectId must be a string or None") - else: - objectId="obj_"+uuid.uuid4().hex # generate a new objectId - if hasattr(obj, "_pyroId") and obj._pyroId != "": # check for empty string is needed for Cython - raise errors.DaemonError("object already has a Pyro id") - if objectId in self.objectsById: - raise errors.DaemonError("object already registered with that id") - # set some pyro attributes - obj._pyroId=objectId - obj._pyroDaemon=self - if Pyro4.config.AUTOPROXY: - # register a custom serializer for the type to automatically return proxies - # we need to do this for all known serializers - for ser in util._serializers.values(): - ser.register_type_replacement(type(obj), pyroObjectToAutoProxy) - # register the object in the mapping - self.objectsById[obj._pyroId]=obj - return self.uriFor(objectId) - - def unregister(self, objectOrId): - """ - Remove an object from the known objects inside this daemon. - You can unregister an object directly or with its id. - """ - if objectOrId is None: - raise ValueError("object or objectid argument expected") - if not isinstance(objectOrId, basestring): - objectId=getattr(objectOrId, "_pyroId", None) - if objectId is None: - raise errors.DaemonError("object isn't registered") - else: - objectId=objectOrId - objectOrId=None - if objectId==constants.DAEMON_NAME: - return - if objectId in self.objectsById: - del self.objectsById[objectId] - if objectOrId is not None: - del objectOrId._pyroId - del objectOrId._pyroDaemon - # Don't remove the custom type serializer because there may be - # other registered objects of the same type still depending on it. - - def uriFor(self, objectOrId=None, nat=True): - """ - Get a URI for the given object (or object id) from this daemon. - Only a daemon can hand out proper uris because the access location is - contained in them. - Note that unregistered objects cannot be given an uri, but unregistered - object names can (it's just a string we're creating in that case). - If nat is set to False, the configured NAT address (if any) is ignored and it will - return an URI for the internal address. - """ - if not isinstance(objectOrId, basestring): - objectOrId=getattr(objectOrId, "_pyroId", None) - if objectOrId is None: - raise errors.DaemonError("object isn't registered") - if nat: - loc=self.natLocationStr or self.locationStr - else: - loc=self.locationStr - return URI("PYRO:%s@%s" % (objectOrId, loc)) - - def close(self): - """Close down the server and release resources""" - log.debug("daemon closing") - if self.transportServer: - self.transportServer.close() - self.transportServer=None - - def __repr__(self): - return "<%s.%s at 0x%x, %s, %d objects>" % (self.__class__.__module__, self.__class__.__name__, - id(self), self.locationStr, len(self.objectsById)) - - def __enter__(self): - if not self.transportServer: - raise errors.PyroError("cannot reuse this object") - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.close() - - def __getstate__(self): - return {} # a little hack to make it possible to serialize Pyro objects, because they can reference a daemon - - def __getstate_for_dict__(self): - return self.__getstate__() - - -# decorators - -def callback(object): - """ - decorator to mark a method to be a 'callback'. This will make Pyro - raise any errors also on the callback side, and not only on the side - that does the callback call. - """ - object._pyroCallback=True - return object - - -try: - import serpent - def pyro_class_serpent_serializer(obj, serializer, stream, level): - # Override the default way that a Pyro URI/proxy/daemon is serialized. - # Because it defines a __getstate__ it would otherwise just become a tuple, - # and not be deserialized as a class. - d = Pyro4.util.SerializerBase.class_to_dict(obj) - serializer.ser_builtins_dict(d, stream, level) - serpent.register_class(URI, pyro_class_serpent_serializer) - serpent.register_class(Proxy, pyro_class_serpent_serializer) - serpent.register_class(Daemon, pyro_class_serpent_serializer) - serpent.register_class(futures._ExceptionWrapper, pyro_class_serpent_serializer) -except ImportError: - pass - - -def serialize_core_object_to_dict(obj): - return { - "__class__": "Pyro4.core." + obj.__class__.__name__, - "state": obj.__getstate_for_dict__() - } - -Pyro4.util.SerializerBase.register_class_to_dict(URI, serialize_core_object_to_dict) -Pyro4.util.SerializerBase.register_class_to_dict(Proxy, serialize_core_object_to_dict) -Pyro4.util.SerializerBase.register_class_to_dict(Daemon, serialize_core_object_to_dict) -Pyro4.util.SerializerBase.register_class_to_dict(futures._ExceptionWrapper, futures._ExceptionWrapper.__serialized_dict__) diff --git a/contrib/site-packages/Pyro4/core.pyo b/contrib/site-packages/Pyro4/core.pyo deleted file mode 100644 index 07b3d9522deb0af5fab313ba9526ad4b6fb9a643..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 45729 zcmd6wd5~RKUf<8_y<4lbv@BbepJcVwvZa={8P90EXlu(FTiu%5vSqiVd478DcK2(& zey{cJYfEh`q8XW47-|d@3`+)vEg@6DPze+^r?q28!xu+3)nZ=bn4+S$_NPch2qp`nsVHfAxohii`jC@%Qj`=c2E5IahUV znzZ1mU9Mhm3teuy%N2Cp?dsiby4&uN?{U*TNxs)j_a^y1H{F-y``vVZk{@u>1D5Y` z^+C5V;3zBU44UF*yyG=`s+Sdzr!tTa?_jK z!e%$U*_ZZPYg^p(7Joh9>Ra8yu$vzC*YtGIO%J*%{|~w98dqKGs_R^Jy_=<{+g#Y= z=I?aX4X(P;&2+g-y{@&s5N>zVce#dhVZWQd+g0yyHwtcghZ+g*aWhQDCijMOH=KKU zdZ(*yPD;0=rT4n(R$WXN_1gHM;q>+{SLM~dsXNos-LASlDZMK#z0XzePD*#ArF&fU zo}_eVS~}vY_v&K$epfA8X0NO6vdpNf?oJ-KFMVL2rX=pjp0xA#i+eBGqZ1OXz28Et64lyJiXkWYc`4j zb=qNZ^5T40X-8wl;<4u9a;r8w*DhXME}m>HgpFb~EPk-rm@SUhba}W{ey!Phwbls7 z8ey9jYk$qZ(E@#FuP3=uYtNOUcDWr=xjix3s~dwPm1cwDMmuT`sc5bhma9wcTD{%F zrDaqeH7l3Gwp|LZ!&a-=ihS$axSq1xQ(^1XuvI!^Z7epM^-(b2wKv~fVJfUMto9u@ zm!EFd>%Pr?Zp|#Ums(-`5*3^Zqo_O^w!6u{5S(ml2&XlRZFT9m&1-vrY^7YUUo2NH zS&fV3c4f|T`1_T`$<@-^VcM4!tPXqh_&O z+AqYiY_hoz_nQcb8dGI%~@T`9(cpEz=AJe~kOc`luU z3v{kH+FVp4%&;}<8vy5_A5W%CFXgp%f0U0e7Ad@%L4ow`V`mg zQQhc+4tjv>!Of2r#E+@b!y~1B|7SqYmj&clK5w1L(V=3 zfpua-l`zn|7#Q4z6d5n?G^}Ytn3V39T#V^cTYfo0`Z<;i;}G* z)39eiZ$@q_i+#e<6`Qs^pzq#Rg;F~UXlB&_eC zSl<{`#5YT&TBFu3l^!4;DRG4jg|&rsg&zKHC~PZiD)jNYlb^1Ktz0FWyDGv=4Nk1s zF!W(C6ug1U3gU#ECjSG3tuwj-`rIodF~pcJxV6#KLLk9;!CmTetpkO2m$2Al50HEL zK%w!QhUTg_wfsiAWPEzE@KQ0_WVJTd^p?x_wsRTqGKK?p*XJe-dn~Ex!8RV!2v{-NU`Z zMy1}2p!zV9cC%SrC^ujSE#LvHigxha`E#S^9y;*;@`0Hn2c9`Ja^b;!7e)__p0A2_ zM|b5K7Ex9+KroUl7f+4~B@C5;=2knByy>w|m=x^NUG=aP8lW4J7=A@L^b}bFtJ*a& zQA3ehqiv18SgtR5+z#$hQvx?Ie^e6`2w@C!m~=oRoGCBW+o$Q3%}l8i_!R7+Cr9~3 zyGa~)BOF>+*k0J$CEVFmc%W-%VG!)<29GuuDq3XyG)IO%>&OVO60DgrVLCcQaaFW> z0a0I4Ksgg5JSmjaBl)IWyj4_6A{sJr z4d+Ip3x`Ikkx*nLQZB*#2@lyR=klnulX`x=1k)XXYXQo6ev$Cqbq@l|Te`LZ$!of7 zrL;=vO1+N@zd9__Hs$zp&97LO$!RRWVN|pk;bU$jzPN>E<(#t zSxoJakRrt-7F%py4doBp!x_K!Y*h^!tuW$4J5Y(vGTd@BWzZvi)YMVewS}qdE8O7; zf{1}2!odCr89$jfka11$1!jTROHkD(MH0o~DDQEX22{eOIG^XBUB-i_46r9MCD`PT za#X3+Ojcoj>*Y#lq$aCm4+Ww;p6)J3CXPv$gia$g%rs5nlBj6SplDW_P$^*%T!*Z+ zm`%Q4&Q1WZ^{<2_q*W?irhCxD}68g+c4@A=zY$DpsV^54kVucdvK_PZG_!8$`lRWJ-xN(EA@fTouK zk~$M?S7J(!*uXOwEW#Y+(gI`>QVxw&xd~}(jgnb)bwSfqDutIzrEBD3B-((Gv#Vgr zdNQ4cL}C>l-k1a^)Y@eHLbKp`l5AYTbGqc$p&>N@SLkaB7wV#ZDO{*;f(v72N>ghC z`2;3_1IC#QQ_mWKoGV9jZ@soQ8*08I!yn+qrILDBDt(N6f*8pJ2;OWpcaw=%^DX9p z(I-RKQ}5Cd2Y1o8Z{v6KNHyYjApUx^!-P!sjGTo8iN92>RnXqw%nd0)!xhjA5}r4* zg&@sBdJVkfwOS)>C9K?1Ml})MQ}EVHs5cm24;~=b3CIoZ=EdL1FJ4D4y9=BXJ5!|L zd()Ots>IM}(zXnZmXR8pt2rvaMsPyOGbFLxP;FL9rQl#vfK9xx;z+|^&dgbZpp?0Jd@>7b*yk0MsFCx~JD>5FK zJ*u1gf=@$cDK0?10G^D8_ShQg;aoM%SGA&}ENIxM-mYAtjn^|Y`$pfbs=8+P_`BLn z%%JXjd9OjVLzhUoaZ10OC&jlJ*`WA zt3B3O{KZCGZcCc~EM3Txa;?7BV)8oWw)(KEWZZ%T?Dz83W~-kYo*p)`X6ssR%|@oo zT)LucM@Bn9j}W$j^HS+^d3LubK@@;7nUNt=pl~ub{Ls^!+<%b?uw8HBTDX9&ouMN)4`J}eoD#H zx+f&DhRlaxX3maHw?~M|Tv)kOnp-GWew4l$Lz8;Dw~)@e#$zP+K|nvukz5?y&;#hL zw;5nsz%hl(Y1Yi?ip%jnZpzKUt%w|RZ7Uqd`?XmQ@7KnV?)}=>|Gi%uKe+d6;{^A9 zZG33nuZ@e$B46YL_kL|$UEZ&a6Wm9<-~_io7dgSV86&_6Zb2?`f^WByVHXnrvekvd za(+z{fOD6kT;9MDzui?W%7p-&kkFfZtdP(f;%o?Qp~5{$XbUAiwB=r_x!dj*#dPk| zgsx@M%&jW_NgT!D&8;mTwBZGfvEoU&Q!C+OJ1XXw6nb={TCCQhMR}eCfOFx3{IIfe zt!%-2vd6}X=D;fT%5KB8C( zyKT`WCWtmdqmHQi8<;37l0QilGbvNwkNH>2!Y@d)@w|X1Y)x`8sfL!l$4?KOsV#)f zrFK?(D!goLO$0;~-iXN8$UsDHpsry9gubJ#0|c#+%rYk9;!gQfbQiLLEBKfgH$V?v zi9NZ-5K9L7k`?-e|1_#RFzw}rAxCc&Mr59x(t5{;@y^E8?pS%bDR$<@kKoe9C*2-) z^V@`H^85Zt(j0&c(X$edeDynHE9pG&OIPkvrwdp}LCmR9ObNR3ve|{U`(ezd>_w$wi?5He(xFn^`6`GlsZ&ioSzj!b5<6hw=ympp${RR8g(L zge|FM4d}2{2+%V^5!1a6E9U)lwN$Ex$Z20B8%yFFGD$og`53oFifpv)~es*mkd`csb4C%N&5pyMBmOgQlJfcg?3QMsV0WRWC?yB#%ZewDs>w=gkRVx~E=sBRnnN(l1! zD`+7~(Ij8bL=tp1BrqAtAVr)b8>8`30(lf--U%}0<@!>E*Oy9vq9F@36na(>8A^UO zSlwaalX0nDXR$dp`Vx^q;Co3rNCM~Ts($Lf^?f}V;j!0Is@W)v*k0vs7R_XH5?-y8 z%%wiaJzWrgxfI$PW#$YO;@-gp3$H3x@XzU6GUQuO`-p~Yn5&3{je^suymHn&tlM&z z8hu0aiA%g6&2*t;t&Os2LeI38LNEFYjx+4eAi85dL^IcaLo*@)L1vepOzczN4G=y+ zhGDlJH@pR1z1aJ>&4s7+gbjAP#rLk99he$YO@2OZHx0qZX#U+W7pLgQU-3(&kKGwU zr*l!HLY{h&&$Bz|*^y+xdM21qA|hsJFw<(j7B&b9Hg4BhN?;C}|3}XfY)A-~K*82s z;+Q7@Zys_5<;>V7W9HL#!%zX%R*20EfQAoVHNq}k=qz_W^b$!N9<`4qTi)7Z2BcVm z#kP`|lw?4O_Z(R;N3KTvV0FTav*LVSrj`0^mQ9NnH65BAOjc42FD}h?c%nrr3(RgD zF*O9ISSL?MGT|v}Tt`Je;X<)Qx{dyb;?%?;OPUWN!&}IPtk@D4Ud4};gu?@7nCP(2 z!6nDoAwwk0LRfCdc1Kn>?Ag7MDx@uI|Ap+#ON&)@T~)owqm1cKCXk{c<9+xnF`Vqy zsF{^SduEoyw%>mtgU(4sxW(=J?JU}#Mc8h0sZrgZZ1X5ed%38QlxK-i%pNy7(B8p- z^CpX9w9yV_qo60iiYFPGK680=gr|tdrKZh6t*(~Q^`)z-?im{OlkuyXPzLMy0>zyL zOm7;pL7WKNvGwSVbSN1oNC8xq6VD&zlk^0^nRjDvTn>pSb*Fv}4K$7n)o^pzs!5S; z2s>$x_J5kvj;y8`$!sR3(wW$znecp3l5CB>xr(^DG*Ahrwz_XJIevBVF-U6g)*x)T zAUzZiYJ@oiT0}^dB!2{6Vi~x!O*;rz^X%I}y--gqPOW3)rZA;qWy!Jun+Z_xhl9x4p~F?~X5Gvpdh- z7rz8Ux1QUP%*R@H_(q|?%B?8f!zf?uYLB>iHYE?3)%;YKpaazDbM4W@QqF~5!9-#* zKik;L)(qah&$S;&c4xeyO~M+^g9cI2td?0cEi&2|;YZIfImy8evpVDQw5=GZ$*7l&qrNx1y?>L4p+h)mo%uZI#G>Jw2 zbTB?Ob#ijTyB6YIzalBmJb&cGR4J)3e%!=9v;X=#lM~}-kG$*^`8mB$WbygQ6K78b z)wNn}spu>CuUxb^v6hI94#7gSJMCKY+t4Jm;9jsTpMEQvylGXU-ec#% zaW$zjKj!h5j*Xu_!)%#r+B!@OcUy4UzL#1vwW}xD$mjQojbD{R5|;l=vvs;wxrCkQ zgVX^zA-{R(^>3E@=ecnCVzb<;nme<_ez?F?RhuI57Ml!@1&VsH;k+uJUvPoD!RL}H zY5!Zn59`Kv=!O6VpbvhOznx%I0ch{;5@XUdxULOd>oAh`@l(1PHnq7hRM>(dcd)R> zw(4{fCf>u=owbEW$ZbOMTt_+**B$H{C=B-xc~-B%>1TXxoQ!An^Kf|R;}NrId5a#a zMC=XwR*_R>#D?3+M`*0hUwA56P0d@;dub1>#ejbZJYB@42FMve6_Ek7GeREP{l!@% zL#RgT#2cwvbcDBk8A@pGee}YQk;{3Neuzt3A)vDpi{l+~CPf^}W6X0}uK_wI8_{Ow zzaoK3cq6Q^+5YBKig>fxm8e=9O|rU2W17OA2+13Z~)sz$eus~6c7j? zA3neYQ3CHvfj>wWt*UL}9dzuiRSEn5NY`iyC5q6>TKDGH3x(y-E_jiD@n`JCP#eD^ z7zU4z*E^2qiE@Cw@#rR4P&cej)i#?PK-*jNcJ*hgn;#aVr$v?Yv(@k;@Uv~U$#Bj# z8Qv+5e*=Qi@3zC^soc6#yM^a*9eW8%TOkL4Hh8ifLz3`h1#8zxjIGOm{SAp9*Vs7Ct@~W-iv^##w$`=j;2y<& zD7NQc7U19h7GWbA(pruK>|G3L*Jc3wfU$+Y(2qIm+EBBjF@PRHy z&(zAMjL~IJ7ptYqJ~bS#@Oqnj7N13K?SioVn1|gT5HeOgj^{jaNlBrfBX9&K`=u~b?*fkGXh{#n zB)SB{#EL&ZoaA9p20kylTmk*w?xa*W@$4x6!HFHyrpP?|7ZDSOT6 zKAtoeKu)4D1}RgcR_C!Xm>g3V{8QBom85dTO`FZV7$U2cNhlRlx~B*Vz+9ys2KVch z_~_K+vFA#`_zMICNUbwH(pSvs=$bk&nEZh143ebWJY)Sb6Kg+K@Cj8n(k|)4)A!FO z4=Rq3#UV=hY2Eyc$~BP^ijGZAOpG5p^URTx&wEB~!ETSKib!m(%H(rqI0%ew=$=^* zY(()sGQxWNL}^}49@0|T_~JDyCHpAtt^%`zFQ~5M<*UI@D=Xs8u>dv}Ne&%hMe;NH zsF0hDCm#|wI?rv_D@#-lQo)9iS(F^ovv=#1)7dL0Rip^@CwuwUJ_tKh;GCq3Z%3Q57mwV7}W}fg0 zlqSH4#Y~x~dqa0n+1s(~Obe4uy$+&D3Jl^61+k5oXd2bSaM9zeAPSZ$$v3ft9kUJc zCj4fCkUd90RTvp;b_D*`(@m3vf@Hj(65QQMu>(d1fwZB*01!3=Jc~|I){5Ckxi}Zr(Q%8mrH@6xx>@IJ06S^M)32f(r^M9f^j;nHldYK9 z;7p+?Sqve%%+=*1)$$@9vJTtlG=7^z>)7{e0yQ_z?_7SzshaqkPF0l?ha`Q^p?5Wx zZVLZ=i@CCZ%6H>jS;u}bH&>!SLaCGr$)5#SL2uAqbf5+TAYvUcIKHqO+nL!gMU=cN zAEE)}g4W2Nr8Sta{qRp1bj9H{#v_e2$vEhh-J?g&9D6qSUan1ZGhcqrYpO0OLf!}K zBi#^3Y8q%qaMp~VqHR9V%7E@iG?4>zQ_ewYn4`Nim=j8#QSu=rA6N34k_VM6D^ZM; z-xG9Pxx18{R-#y?;JT8JDESs8&nppX`)xzg*n@8)xg}mKGgRMlE3@1NqU*t|5|+>f zKceJ=O3d2x@0I&iCBLTRKagbeZMjtpMwFl?svo_Nq;IgVu&Zlum~97rgT1yVse7tu{>oiW!LL)Xb8&*ghMy1yiT?b(?r!>u z#`POY6fl=_p-k(NpHEg1kcIC*t+5)5^4`sa^iE!;`^jXG^l+>w` z95uOJ=8ezTuw)67iG4kz?`J=3P+F4WpORxS3p;$^te#^T?`ZJnj`iMP|;lL__ztHd(1pYl_RuEx9 zy-kKBAA~;I=cigksUx5x+4Prh>bLr=p8?LhZ>q1;wO`3~S<@^yS5GU~ znddzByo_tR%e|<-XLIYL#(XTF$hjSiTuP05 zXIo;}Ewit-%C#sw+0NM}`gtIxVu8BSiFNhbektxoN@;&h(66XOoq4PMsBWV&(SS{lKPmOA-y&-iMIX`IAj`JEq z@SjynI#3`2*WN>Nt0AOx%VcF*Z1810ED{?0Hc2NI)V7U(Ma!pdNLAIzpBuXrh2B@F zX!)dS3m0q|#S*jXUZWTGWMAe_wXyjAq z>-F366YY4zn&68RSNPdK>UmLCI$|oyzJa95KJi4FMGzG*+$t8yEdX{he~3lZjESmM zFYg!z-9{S9i3Yz(t-qz05xMJktcT&)EQ%UQbb?`n$KPA5vYu#H3oxbrRU-eiUm zMs!YM7Nq=HvP>&90L`!=#yw)a#_a*jfL@?&zGF5`_%CLd57)cK zbZ~Qy`Rc_!h)aT{?~;AmBO?&2NQ=cc|56JfFOEc#JxG;AS{bRJWPqQ6-eoe&oPL_f zBP`3C_bPlwv+l!u%ts-yuv;NJlEgps67e{!5?c+=HV)t{rtKhwqe$$}=@pTHl$z0%UlWcpqf`6eKTF&6Nlsu{A=SjTXOY>{7 zkhWxAtj^mF{g&8hWBD< zDT6;G$p&D#)f;@P!BA3AaI7oL3eif-$lV9PA+`^bdIer9xidzlbX zP>XLa!ZX@k1>f^$iHGr4xzrCw+^l^vK|yl)rMis;`0Z4k;D7VJZ0h|bMXOD{&7XX4 ziVU~eGbcNCD!73UI(EPSJ*It6a6gL|sS`-3nUlReHsVWCjWDdwatoJ%|JsBC9%E3` z%o}v-S(^w0b^mm9beU}cs&Xk@j@|{~BH1{0r?ZxYi=;$TJ%(U-E!g-&Ir+X+wRu2i z7%AK`<`d?8;JY;{T3l)gz@iC{f<`Ho?FdF{6pQrg{%ojk?TGt7R1W@XMOUrUtH{f~ z!Q1aqUkAt-6UUREQigsCjWVdGD=5RT5qDDmKRP2{n(T2Q5!t3TLep`>>C%)(UG}?> zqwSIhamL9Xho8>Qd6rAgTy^r$QJL^%@1f(QJAY6fC*Apj@;K?vAC$)tHvXVIwvf>* zXXWuAVc`C(JUi@8ui(TyTz+gL^C#wUnu09ZaL=s#6>(;0O&&hS^%D`;)sCCq8gk+qBH zBX=5xYCf02J{3M)79+IxE7s_MtMrkGZE0nU^Im1~AT|&f3r7U&+3>eJR-v%&!g|G9 z(W3kDG%!+_Bsj`y#+K!z*|d=(tsk^X zX1(;4;`iAxlL~DAH;TNazcByOkZa5*Z>54ixou+sa>fn-fJ62FOi{A{NBUNdWt!jM zT4)t^UR;Ave?9}6Hn=_6Hi*LEhf1ivf0$RY*Ff6{scrhCG?*Ty&fl100z0_bALA$731M&D- z`TXU5_MZE+UqBFoDSZ}j@v}05w}rFu*u;3h?fA2=n-P!y;VyUi{ap-4&9lkR&jknc zC68$`eQ1Ov;fY9ENySH`6LZ67`#7N)p4M(6O>wwBUStlcV^g1fo`zV+UDUS|M1k~I zg)lr}UqiBr@#DcCQ_Ixs#OZ!D5DG+r68rjP5fAht-@80;k<;$9xzA7#HmTV7i0_SY z8a3tO74vpX-hRadfn_p2rPcA%b7^6qNt}D!lJ*f6_l`t+qb4q4@k;rh; zzL~x`UZ$yXMf=Bmf8+D`_^{MSH2MHH{h@q2mCMA` z9xk7qoH}C?Dvjpz`Yc-TC>4AZ{CM!-k^?$I(V}OYLg&O2fyUC3ou0=3>-t;{2-ym(+q*GCMSIs&wR-w*IjVvRf~a z-6QyiswbMCwhuhSE5**n$VW+wQD+~Ou-VT*+-5HZsy#!n;bl)D$gm~M1 z$P4YWA0`c0vWg;s(D@;W1kaFhg?{Wr`h=Z+TkY>Kd40ZalYNG3SJ(E!c5<6>Ypw0N zx9eUUJHyml+oipKoGa1wSSBNDt&MGEDl*{>i8o7Y9Q?JCUnJ?Q=xSX!uW3-Pa`DZ7 z!y(wRYcmD}ufrqROqXBDjYOnD^FPSX7USz=V1$sTcET5N62}3@-R{jV!ywrN$6-4b zpX~J+FFjoS9^)`LB~dOfe9|#vus@v-zyl?5bQlaA8jhu{bnL_<&;KwRYI4p1TiCuw zGKDt}a!ZT{`Jl_4cvf7LV+M{}W0vY^+P3=zJ^LkD7-UJXa|}*Ad-+T5@^SrtR(F3_ z(x`8TH=EQ^&~6O&mnmvmrYYY+I}5voVEh@ zthL~d(gyWTeU5XVs+t9qZ(5(6DYA79-eSAgsIF6C(IYuBC22X#nu~+6Xisuz%;ae8 zN?5P&FWMooSBQ;}7`j}X*Ad@*a$(s%(^k)h7HwzNCjBRozv-cC{?on~Mv6}szvhprj-%t_6~)>1<923E@x3!DXj~Dv>dhmbUUnaGU0$y~gk{p81S!d*xXeju;@GgAjiaWzzQSQrk01 zA&NRv*P!gA*iH~;8rxa{(7gSjLrL>o2 zAUnG}C1A$q6N_y8Nv@C4TsF8>W<9p*Pt?l`7pvu`KBD1ibmkEkdsctDuTy%J%qJij z2ET<5FXoHGOI zxQHlv$?s-Ou^vR1iwT`F7HNuhcv5nVvlm};_$X=78-|b~`9disBd=%LE7he6HSC`N zee-qtFYzWngKL|^?h~d(uHeOkz>3Gdh^-7BF4Tr?V^quNzg7FvXrP;`3@*Vgy7#RK zm<0E6HDMxlJR3S)PiC}q9z4BC2UX#iY%i}G>99SY-82#mDUlrFjckn5gh!KVv$!#P zp=l`4c0UW*4B1ValvR}6bP9u^jqLTVcO5cWx5bTZ>sp^hUT6W2T?YtMOB$@95$Z4? z?pHrL%N?mrQkMkS+V2cl>7(q#_!3#HwLfIrsR@SGH*n)w7AO@-)T_B6i197S$P)HNxs}fyK+5CKx_MzcK;k(|ReO+Cbc35n$f34W+_} zO*kSz%vH5sFz{qSv{1+ioVFH!vLmjRQX8XN)!K~3rlYi*a*Q+(FIsG-GO419o(vnj zG-ez2gC(-DP?9=TGGxIzfw(qiyS z3_JLDNZ0Z0t*troi2R z=S4Dk;DHyxe{dAQn!rH?K-l5iv8T+q6Cy6)0Z`m9yPe!;h|>U>3AH>`K#z8Ljf)Z)XcHU3$S><4{`5?r9&85jya7Mo+{tK$WZ?$pT)NQ`e7as)iIW3x$A=fWu zbr&Sl7HlgeD(=3{{VA3YJZGJhrkgTRU^r03;vbaux#i0}fGEqx@*tS9W-L;i8FaI( zE{}NY-H=*CX#wqNP#NDEPuVvyT0f?59?WCn_SHD*j5?ehVm1APO7B>4o)`rjpyuD~ z&6JN6=uuiC3S$j04&R2r2@7NT+Pe$!YcX0{Yf9hgwE!dA<*B$?)J(vL>2^v5K2~Il zyM!8dp@7wf!&fln`dhJ47A;TMz;SH$fG_~iCR7CbDkT{7lPQbGvE3cMQ7D{8wd8;l zCUT8~o_sTvXyq z@$_PYwF=9eO2d?{QTv~!iS@+Ge42Qf5sR1knvPP_i81$UGFhnVfZHXtdV{;h;(DOK zT5RXn6!e80Db?_hh)GjR)lZkh%Idx>WN)J=WfD3`XPOgq#DeJ?7>QT6;Fs z!s*Mqx?b|PUSDskNx@m3!X0q=Uv;31UX#{n z?#AxqM%>ss*LtNVMWy)ZLyndN*`NiOxGs;o8Jbdy<@L3w&ae$fMe=XLva^k!OTlKM z_wB&u^$m9N(Fg4KrairJ$3Qs`MJr|~;{NTR>YBMt=ASUn3EhW6u$mmK)XPHc=@mTn zxoEt%P2d<%#u_|j>C^IkK>1pey&vT`q{&_j_|o@HWl4zsmnf|)2Kx6bbQy^<)tk(0 z5X+X@_~yU&B!QDtTKq9=UgBK9P|>VZmRc(eC|B6`fojH%JO!ZoGrnR!n^Y__gW!O` z>Vn|fG{1puU@&<#HjuK)Rabh?<0*nxr0c+EgT^saz`jqt{t=Ne|HL3wPUL`{a%GKo7`l2VU6U zW6rWf9W^oUF&;8i!DN-OhwO-(OI`%LpnLq>>gik2Ag_Budv&fZwx0TRNfH_(Nbny1 zq??aCTq=x%gING>$sPuJ7UM>74Bv7BnFWNezT4R>!FSS3@Kq(}m1u~jdHeI+0APhw zM>{B9YFug{UMB%0af>>}60g(xTPg(KPEjN|@tu2_^{!>Q?@jAFdYD%HH=5-0I_1A& zHI;fyykPi2Rgydv{D2azz8#qqd_OrGv<^i&eRAT2oi!2skSg4-P?h{?pHeiE5}o>~2eRv&@6CjF zPFp)_jKCkLT8p%h!i3-pdeKjjn685PmOv;GBCo^-KdIY01uOx>p(>dH$hg{d+5V$d-$c>_3*a+k$l5K zaYiNj8`31QTx(4Mp)M}mh^&a*%J`^+tc#3!tbjM3xRff&rGJu86rN+pH%M0m2@k~2z}Kg7@Em0h~b7}P)3rE!V- zmHR#=89OrCXr>QiZZGSuc+#lA-e{hYtjr4IIh#F!f+MwpgwGBzvm#ucGJS_~r_{j- z5+hFSi=MiDj$0m{koRTVjRqxDp`y7L z=hWO94frcE0BUQmIh#RGBtO0)dPw6$=>k?!*)AV)4I=iqYgB=$!m6?jlL-?BaDygN zDCP~&v5XNnhW<_@8Cp}~t%)Plyai&&PNV-+QXv;iCgftzidheoS)-#SIlUh2ZT+^p z`jB`kMiuasP8`3m!^|~wYLnt;c!2&Qafo1D{(1Cx6JTGbyU-etu-`qQ3y>MqlzCmG zF#2e;As3rRYIqPhO?YU&KA1r82(=jzh>_>1;u+a<^DzX!xLR!P8;>o zV69=wOV0XkpSwsTcY?)nn{w8A9m_JGA+6QR7ft!1E9SoS8c?RToKXyNY!S`^nE9M( zY~;^-onuind)UIhe3wsh_FZg_F5_c1cNHiv_qM4r)G7u20(~aRnSdLQH92LK$fRVx zh{zFU-c%v+4@>G+_?fqoS4SnncwuFSa`%wfE0>lK5Cx;kJZ8e+TzECOqDtzCmoJYh z_m?XBQzb?vj4XUjm)#`hzsQN6{unjyueZ&*NyqH7RBU1Apz(@DUwIOwP>ewEk=x8} zBFlH_#zEfS5$xZ_i;Ije+DSJgX(;+9FR#G-NS`@kPj*nsWcVBdc*jiBLgxB zOW}lf3*%En7Z)@PWXeSbE@(Ma8Dap7hLdyG@_M!=;Q*(t@ZZyknrj%llWo?5U8#&C zA3;D;#p^1Z5{iR_7)1F#=+DG?3ZINSg?W(Df_ueq_LVj6^3RAn>ro#f@+k(8qLxl7 zcr_IBj6p8_v`tiAp@e%_j0QXC%)*MDx5UZele zAK)V9P)B1(1$JZ5#I%@#HFwSQ58%e2c{DZ|w=NjS5mbHFGRUkP<%kFhOj;hU1whjj zwmyXk8>0{{ihhp?PnZ3KZH&uPThAHdwN-#nvdq!7=_k18}vFKt_zc!}{4Q75qRW`}zV z#|5RS=N;_hn!{2=ztc~=J*ZZ6EV22ZZIQibV+)vk{EcNHkWYZVp8BzRqR+PhM-zTe zcP9ADTkRdwn)%Lpp{*aYn#%$D;y5LHP=4oR4kdEh9KMGP#7h%qGZ|-Km{1?{+o{A9 zaAizLp0O`JYZ)e@dr5EiMdy(er{e>D#;JJo2b~3s9^k@;iBKJ$vy=3Z{TRs+Y`6QY zv&QS^C^DhNTPtpu-~oJbD>-;qas4qlvm^TfF{p&6SDr%}yoja;yGS}b_K(nomc}ma zQ-@D%DpVw`=_jZ$Xb}?l(chr7dQ+U>C%Bslf&<%b*Sw#i)TX^Rc4NAc3 zGVo6aq|w!~VW?Jz*Fm<5*WRwnGAg8Nu)-ChM?2j6({uqzCYkWq^m2mI+%e3?mqdU4 z0m#3ooZ``gGs=BTNl}*yT?;gzK=IhYLrU~n0JA#;Qa^$ZE0JpC>8=nscU-gCebl84 z_$;@I5V(%FKM3@hiKMTGz}x*BdN%ZQ7w+D%L1!Yb8C*AbZgBhBy9e*wu*J?$w)2!z zLM^~G)KfzeJ>g_VuH&e@pQYI3t||xE+d-U?To%f$OT}`cmQ#R0RW9z8Rxg|0UY+L& zF|r6Qb8gC~YR%Y5c#>&~W^zNLu17k^)Js9}^R}hY!qUy%Co4lzMf#03WXg3_QLwUC zByYBimvel4pRM+LSVa%Gn;1^ESgW{BqHJiEG3oC^+z@FHan@g3f&FB>xZ!0BmIAsj z+b~8QvrWT<8hLW(nU`C2M5t4P(68L#1X3);Z(Uzr>9}3+k+B5HQdo!Kh zo*ZYT&`?Kp1B)jzKb*V`9vg znixGi$ePR49H8ueWMiH=GXibm3@+*knF&`uku0F?nk;1&zN+DCsg!}6*!eMvKW~ zlx+3d-un!FjlNDFp)b%GuCzALlP--U?5sGeA-Ny_8M61!;o#HXe-=utK0UfWr7QeJ zMho$gY)4>6Ogch$#qUD=Cd7x2U7>pdyUDxLdV50e3hX6szxD12y|?!6x87Z$57yp6 z>+K7DW8Lq~*1IS4;o5tv_3jIOd+oi`dJlyDLYxi1Ls&>} z+&lnYDy;cIAh(PlJ;FSo24msv(z|Rh;2RuKE6ymsz{*90H{)MW#gGA= zwNso)w!G^M|$@-U(K>!7CbahGwhJ*clw>k>vrO*YHp2^yfzzhI;Hq$%tw_n z1eiFVmb@7fJeD%j0!Ih59Q-z*t`7#acjFSP6~cni7hN_?^a@h?^}mv0K!QWSwIm&B1fZX~ruJa%d7DaTvo+~k)r z6}oxc5W_h|4uC$#K#DI3$Gp^F9c4AMsp=A0G#4h$s4{Q`2^b+ZV?j+h>GY%2!Afnd zwqzd{V8SV7NP9Z>wiO6Bdr}uLKfOv@1j$#+>oW`P%fGt6`BQMlTcv@l>SYT z)kfO~5h@wgEU}cN-vy>Z)+QyTVH*4qG(@Q|voIy+XL`_Pa!Ty2Qzw_#Z(F2X*KSbj zwXBStZV2>83ZxvpPJk;^3xG-)E0W7R$lwc3hwMt*Y&s%6U!Tp=8KK}RUvSMYxLOsQUj@skV3q}ke#Xr?owUAh&Q?G0 RB*Pst{odER>AT#6`kdmK5Rv-=|my}J4n(XDEtu{b>Pa*%BfPJ?RK>sk}EBD znVq2}1~%%}MS!+I(>Eyk0R7iD>Lc_4+H>wKNhR@bt3OJcquJS++1Y!~z4zS9`oGs! zfBmn&hl#2_b^N{lR4MyUbb3ndp$$~8rm{edYHF{h0$$fuR#$s$0nIL)6+?uW$}Jy?3K>y-M*d{lgp*i6PxA({b1^bMXs-Pu4-(RnNvB7 zw#KXuog079?e>bqcE&QZQitCZU@lcDTYl-^nuUm{=H!HBTd#^(FoiZL7d6llWkxqX$()~Fg0ioFL)y{h~g|4>^UfI-{ zt#OKIA6|4jafy9$>#{H+b1NX$YJW%RMot@tR z%mH1;lZJ9Nbs!#!6M(FmdfZTtF%1%`seS{~YEPQ#F}zSyxfZ=pz>6iW4={J9UB~wJ zPiREo6#9;f;ZYXw5EObnaJPsin8LMyXgS6q6l$X!`vx}OzCfFAr&4orVi!=-D>q5iYTzs6w%L&U+|^?arecc?*sru?y-i{ZW_}H+pYlif^*QP+d z2e`!E93Q8J2%ZOG$lu3bOKVXr!L*<}igJXbC=yN+MWdoO$#_Z7M-lj06ooXV@VjhC zS3-J<#k)xFg~qy&R?6d8@~~D5&Z@DpwsJPXm3(N*H=pAEK04;P)M;7R6DTzJGuX8d zC^G7$8Hd7zrjRL6Zm{WPWmd?#!K|NwxPyT&$=x%gS`rhk;aL^!nW4LiLxRJT%u29W z1vv@NbY4`9_}914afrSWl3|ZgZAMG#GR_!GJB`5i=>NEQ+lL)?#lhWvgfQwTqbh{? zz!*j3tW)uPv{5lp@mTkYf++0 z(}EJ8z;6jnK3rDiTeG`xV~`eZRBr4aSU6`WyD=yuQ=Wxu@{a54&Z%M>;B+8qa2{m7 z#l#_%$dT0`W$7^!qF>}Etmp~}f=;~)^KOrJn^DJ%9yf>^0! zwqv=e9(=y3@P9X-r;*5URiHrrd`JhH7a(^tqNu0FbtE1%upA`6@9t5*RVC{EH2E`$E(Ja`vG_><3Y`~PfMvf7jJNl+;+hu-!C zfCb(HbV2s3xck4xl7-cdaWKouoocPuE(90qtHEo*wp2$$jnVLRHa}n^wtI!WHX5PQ z$Qe-5bJLJHSNJv?*6zHWo;qo!mE6ErlEGFTq`7(a%zATWX=Uv~Ex`@>&@l;h=Bt_K zQs-|~9l?OPxqStTZ)Gt;u`e#7#K~%0>1(n`r+w+OqwkUn`P!f_=@``dq@u@yr0}QG zY@xu4nhy!te5satA?bBuQq*h*+M+bs(VyB0JjZmttBVhjW+pMJjzwSgL-jY-c5D)q z07wqTD6$zn0&hS$4b`Q1u&MXKZ8D*Ghd2l=I2qkb9!|?UTb8 z-f3t`>Buj7i)j)_vXr7kLPl^2W^b!k3~BB;}s~(UHEjp?+T#St*fiiEc@j z`ZVSucj60=v)F-^k3vpzG@*Wad>6k%_`>P!8KmwM<%2Xi%uGlL3Cj{ved^>TWP0*P zeCIjL#&ab1Iwqu&v7p}iu9qQeW{{HCl9ZHF z%Q;{BO`-Fx-yH;DQIybHk>CriXv1F6FQKXpcA^cY4RX(SS2CC$r" % (self.__module__, self.__class__.__name__, id(self), self.type, self.flags, self.seq, self.data_size, len(self.annotations)) - - def to_bytes(self): - """creates a byte stream containing the header followed by annotations (if any) followed by the data""" - return self.__header_bytes() + self.__annotations_bytes() + self.data - - def __header_bytes(self): - checksum = (self.type+constants.PROTOCOL_VERSION+len(self.data)+self.annotations_size+self.serializer_id+self.flags+self.seq+self.checksum_magic)&0xffff - return struct.pack(self.header_format, b"PYRO", constants.PROTOCOL_VERSION, self.type, self.flags, self.seq, self.data_size, self.serializer_id, self.annotations_size, 0, checksum) - - def __annotations_bytes(self): - if self.annotations: - a = [] - for k, v in self.annotations.items(): - a.append(struct.pack("!4sH", k, len(v))) - a.append(v) - if sys.platform=="cli": - return "".join(a) - return b"".join(a) - return b"" - - # Note: this 'chunked' way of sending is not used because it triggers Nagle's algorithm - # on some systems (linux). This causes massive delays, unless you change the socket option - # TCP_NODELAY to disable the algorithm. What also works, is sending all the message bytes - # in one go: connection.send(message.to_bytes()) - # def send(self, connection): - # """send the message as bytes over the connection""" - # connection.send(self.__header_bytes()) - # if self.annotations: - # connection.send(self.__annotations_bytes()) - # connection.send(self.data) - - @classmethod - def from_header(cls, headerData): - """Parses a message header. Does not yet process the annotations chunks and message data.""" - if not headerData or len(headerData)!=cls.header_size: - raise errors.ProtocolError("header data size mismatch") - tag, ver, msg_type, flags, seq, data_size, serializer_id, annotations_size, _, checksum = struct.unpack(cls.header_format, headerData) - if tag!=b"PYRO" or ver!=constants.PROTOCOL_VERSION: - raise errors.ProtocolError("invalid data or unsupported protocol version") - if checksum!=(msg_type+ver+data_size+annotations_size+flags+serializer_id+seq+cls.checksum_magic)&0xffff: - raise errors.ProtocolError("header checksum mismatch") - msg = Message(msg_type, b"", serializer_id, flags, seq) - msg.data_size = data_size - msg.annotations_size = annotations_size - return msg - - @classmethod - def recv(cls, connection, requiredMsgTypes=None): - """ - Receives a pyro message from a given connection. - Accepts the given message types (None=any, or pass a sequence). - Also reads annotation chunks and the actual payload data. - Validates a HMAC chunk if present. - """ - msg = cls.from_header(connection.recv(cls.header_size)) - if 0 < Pyro4.config.MAX_MESSAGE_SIZE < (msg.data_size+msg.annotations_size): - errorMsg = "max message size exceeded (%d where max=%d)" % (msg.data_size+msg.annotations_size, Pyro4.config.MAX_MESSAGE_SIZE) - log.error("connection "+str(connection)+": "+errorMsg) - connection.close() # close the socket because at this point we can't return the correct sequence number for returning an error message - raise errors.ProtocolError(errorMsg) - if requiredMsgTypes and msg.type not in requiredMsgTypes: - err = "invalid msg type %d received" % msg.type - log.error(err) - raise errors.ProtocolError(err) - if msg.annotations_size: - # read annotation chunks - annotations_data = connection.recv(msg.annotations_size) - msg.annotations = {} - i = 0 - while i < msg.annotations_size: - anno, length = struct.unpack("!4sH", annotations_data[i:i+6]) - msg.annotations[anno] = annotations_data[i+6:i+6+length] - i += 6+length - # read data - msg.data = connection.recv(msg.data_size) - if b"HMAC" in msg.annotations and Pyro4.config.HMAC_KEY: - if msg.annotations[b"HMAC"] != msg.hmac(): - raise errors.SecurityError("message hmac mismatch") - elif (b"HMAC" in msg.annotations) != bool(Pyro4.config.HMAC_KEY): - # Message contains hmac and local HMAC_KEY not set, or vice versa. This is not allowed. - err = "hmac key config not symmetric" - log.warning(err) - raise errors.SecurityError(err) - return msg - - def hmac(self): - """returns the hmac of the data and the annotation chunk values (except HMAC chunk itself)""" - mac = hmac.new(Pyro4.config.HMAC_KEY, self.data, digestmod=hashlib.sha1) - for k, v in self.annotations.items(): - if k != b"HMAC": - mac.update(v) - return mac.digest() diff --git a/contrib/site-packages/Pyro4/message.pyo b/contrib/site-packages/Pyro4/message.pyo deleted file mode 100644 index 37119794c6d8094a2af68d3b42efa355f1119d8a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8857 zcmd5?TXP)8b?(^(78im9Nr{AH%2v}d&1IPa5}A%FjHD0(ArTgCnMFuz%l6b@r?ESW z-I?X|EWu?FRjSY_Kcp&8{sZ>k$X`g6pZp8_l;=F8DpjuJ`%cg7E*BIWP4Y)tguS1=Uo&$lP~j?ssJF4TZJLd+G@Lxukxj)GMXFZhn`WYt-Mu_N_&+c+fF=Fv?PW5@)Do zX`Z&yp6(lK!?synT-;~phkAdMchW@rrl00U?;UncE4Qm!Z={$Vw>!B$9Ow3(`2U++c&7Q@HSxroSXLT|DANSJ<(DqLn6S zPI8;q@MN+q&1^o0>W<^hUPPVT-CHfVjY%{nftZSpXIRX#AcYtoHMWF#JbeBQr0`o* z3@ia+LRk^Lp@j0GbEp~`hGOSc8@fRwbTgyOtm@X3nGxlo7pj87p`i=vS5W676$Xhz z&Cx}vK+~cFYI<2(E-3>|UuK;aa9JKf;R-3R1-ic~Bd;pIQGXj;+6C->QR7E+a+1Z+ z#J6aSBc4#3g;=bHDDyB*tWJ+~sE^|3Cek*3VRXlYk;(4rFwyBCkJBXVX=`#Bmn12) zf;Qdi43lGfSC$TwNar1B8HFB%qh1EY%CG)lq6}ayy49fFL#e+VJ~A ztsa>>)?{(mi(k04NWX)k7sGcT=~;8qF&T`LJ6%7|rw+08H0h1N zUS>^nSDWW8GsuO7+mCkEH*UKf-FL*vbY@_G(RhQ;zz^1hlnzZR99pB}oa>_&>Ci%k zg=lpYw~fu=CS--aqGZFeBwygZ|{HY@7=y5Q_%DgK4UFDGns|R zr~AsK^3lM2Tl-NjY}@G$Yn~2G(lR<3_76=~c5!(TWX~k+yi+!Fd1*eYhhhWuvejhZ z{S43HSnyUmlDeM5WO`yfy~gzcFv-(ZDZl|XUNvsmOV3P7i)D)>(~za zPMGUQIryOkmInO70HxAQ_fshU^@aN32rB9I(v!Q+s7#@u*VHpD7P0$cZ)N*)i0eFk zk>BAcH}))hVSZPb3|sjSVwv_9pe2SEJWZKTsXM2-vCZWvqzMU7l90~;(0InZGuH^$ zOu3>kTdbPV8f;!#Y~iMm{4+@23V${Rf8y!lR|UMJ!(OBZChLR)i$(Kc=FE~}AM+rV zbCjih2qi^NhAufFS|Kc;e$t6s9epCC(9e0p9^4|qa4nFoW!6b-%}r+p4n<@JRvrLp zV2YT*vj8>?0gwTkwvK^Q&TwG!eeMUZN$`{6O=6k34lA;8EE zGp83>?gUl}OC>%M=ma7ml0s*#6bfzSnm{qZhGGdKBxNaE$?%l+Wose}A@1ZRTtI4B zK0#myJrj~~1S5!#Vv`j+UQcuZk*^u~_zX#fL|$r5~qX@XWn4WnIddKu>5u3 zbSiYi>tmo!Vcx|A0=9z^fUq^aBZLs}L7nq-Cnn}R;ROWqDo;uCRJzE9#1X6nJneD@ z3==#H;wZm>mdQqKe);UPOtVX9ym`+)!cQ#4f80YL!AOKb1iv@F0bvi($4gJWR4PDa zMk;uml}bgu#2>n<3f!U6t@4y-3kFoBQd2K$YVe4G3ft#YKCik9s=KJNflRr;7>;u# z()=XB{+Cp{Z0cTQ_rqUYSEuN&$U1XeWaXC?_4HM_9HLC{;3zMvnd-4p>AwqA;k0{6 zt)MBXsEU{UNkJlAL?Vvm@qqF4ql&tY75>RnUCF8vDDSV%SqQA&yj*@6l!bKSC zQQQWfN|BG?tIe;4DcY?wlVM*5;;08a`ZNU5{?7W(f}PDqWBtp`pt055^d%mZP_u}Q zHyMf6Nm(e;Bqc2E+x7v?ku1^X#HJmClN|`F3{#)gQ79>F2emcCUu(S-u(`G!A7)`T zTKggk`{pFgj_sO@zE_XRmbJ$&POxhqt$w`Lix1ZZ5})%4Xl_0ngso%5@0^JMd9Cox zRg5CzK@cZ#9t4k3x9^}(-a_$v72nI=l6T$viFem)kyU=kw^;c@RB%P26*|N7R2Q}; z7VHeVJ8dkmX|Y!<%gO?+@MvCLZ%CZKYwv^HC%5eCEud2^qoww0JZtaAkiBU z(0Tp{q|05P2uTb=-_cZ{V{x0q-60v>IWvIssF->ctK<|~u;XrLLEw(7K-?fFtCOxN z$aVeV+5bNzpQnx`;A$cHig*E~&XYhS|7%p7)7|>H?1#Gsso9 z&fB%Kb$YO%+rGbdu(z?d9X#Ij8(Vw3&g|znkO+7Vt>k`Q20lSQY{xCZOxVw`l?(b2 zXXIQhwC%19g3aRL;z;6zC7n0rRg}hAqr@?%ETyNsht9kBKw*fE{{To~^BOpLRn{u+lAYRE???GW0fx{k6-qZKh69|(>c|{#p zRQ7^8?LyZW@2ST5heFT7FM|;|#9F;L2bZdT_87zg(05rA37{kYhZy7EWcR3n{KBY;XZy zKeot&v^xF=T>UPJIFam%i~Di28zy!*7^E4qE$7o9f@%_=6og9<(PO$KS(n3GOq=>H zpoqbOL~s5zG>Q)hdJPkTmQQ2$Y1{q}SlnX4Q_RJcq$6YtKeIsUlJZ0BPMY(uE#FAm zh%X@m3>dylMXd9rDBekm;!A;lhgGhH32(}$$PVxA#WDTQc(gpfRrNjZs`q2>2Ff*j z-jTSlMQ$kp?nnITeRx=LPyyClKvWHv$M~G4j;tw5*!yvL$M_vx z=1kXkB5xOPHOLl$BLERPaE?p2Sb^=YU@?JJxfyx54dD`m_OTDcA+tXU^-FZl<2gPe ztIPE+f%t4e9PUF>1%W9abKGVpvI=%7%yWx1xu#WBfGjQ^=hXAIIGzl^<}&LQcPDHa zfAC2_JvM%mK!OuPV8ivWx#%Sf+~g|D^y*JsD;qz?5>GFCPjJtI3@C66lD7w_1b9UN z43KHZHI@CrD`fZ<>#iLG-ev7=^$K_nNV!qrcJV4ff^uNe(;JoKi!+1}(TxdV({2{B z-G$s%2s@?Rt`!djxbZCmZOr}`Er%C&{`jV4;%5Rr1HM-%5y)kbbQ^sMv(*W*A4r3` zn?5J5x0u3pXyitfQ|WMsPIr+S{uJ>k@_QLFZ2&GHP>~2NG6q7YXH#ZnkfWJ^+BBvy zqWzY9-Gr0JMQ)f2Oy?$!fL@Z#oD7vP>@U6eT1ssq0xi-1eT{Q#f}Ow(xhHAdg^(zs zNTYFNAa`z`p{gbDm2#4dO$nR!yXcXYV>6<_aandrrBT0c@+@xEc`fumLm`m?R(a_GFIA*9N_bnHk{0$#@LitR=_*YO! zSlKYGA>MP0GRP9Q){Bqsqcv!8es679Ilx%EHgK%$ti zh-Tpu4>EjW6E@%>CV(Bwy3)lggv)imA>8)=9vu7zA4`d;`5U;TYwvw;5m0?iLLXe< zE4;+l(DDKLu2rr9%-==t%}Na;u1-Q-rX>W-yQm;40;;$$Xnchyj7yOEs^T&J{ae9& z!Q)rVHPxr$-!k^e?g>HxtBXkYo_cyuJt1ldOgH4RXrEwBmsNINE!J4%{3L#0gdyH| z?zMq-DH~m83#XmXB?w&#M&8UIDY+$yG_i z08u*71HnY>SJ<04kktN9lVB3)aTkKBwI9{I@4;V|y;^1I(n57%2EW&rm#g0L52s!! z6R~0%!`fGU5DtpcDDc=D1`r~q$E z3Kifr#x{nV%zwaco(4OOFN2M}-QCTN0|CXU=e@6TV)0bV7wcQw!WY}Nb|3G3wK;zD zHyi)BeIOcT@BY^Amooi}?e#Ak!RF64Hun#Z6%Z(NZ5w+#`=(3, 0): - basestring=str - -log=logging.getLogger("Pyro4.naming") - - -class NameServer(object): - """Pyro name server. Provides a simple flat name space to map logical object names to Pyro URIs.""" - - def __init__(self): - self.namespace={} - self.lock=RLock() - - def lookup(self, name): - """Lookup the given name, returns an URI if found""" - try: - return core.URI(self.namespace[name]) - except KeyError: - raise NamingError("unknown name: "+name) - - def register(self, name, uri, safe=False): - """Register a name with an URI. If safe is true, name cannot be registered twice. - The uri can be a string or an URI object.""" - if isinstance(uri, core.URI): - uri=uri.asString() - elif not isinstance(uri, basestring): - raise TypeError("only URIs or strings can be registered") - else: - core.URI(uri) # check if uri is valid - if not isinstance(name, basestring): - raise TypeError("name must be a str") - if safe and name in self.namespace: - raise NamingError("name already registered: "+name) - with self.lock: - self.namespace[name]=uri - - def remove(self, name=None, prefix=None, regex=None): - """Remove a registration. returns the number of items removed.""" - if name and name in self.namespace and name!=constants.NAMESERVER_NAME: - with self.lock: - del self.namespace[name] - return 1 - if prefix: - with self.lock: - items=list(self.list(prefix=prefix).keys()) - if constants.NAMESERVER_NAME in items: - items.remove(constants.NAMESERVER_NAME) - for item in items: - del self.namespace[item] - return len(items) - if regex: - with self.lock: - items=list(self.list(regex=regex).keys()) - if constants.NAMESERVER_NAME in items: - items.remove(constants.NAMESERVER_NAME) - for item in items: - del self.namespace[item] - return len(items) - return 0 - - def list(self, prefix=None, regex=None): - """Retrieve the registered items as a dictionary name-to-URI. The URIs - in the resulting dict are strings, not URI objects. - You can filter by prefix or by regex.""" - with self.lock: - if prefix: - result={} - for name in self.namespace: - if name.startswith(prefix): - result[name]=self.namespace[name] - return result - elif regex: - result={} - try: - regex=re.compile(regex+"$") # add end of string marker - except re.error: - x=sys.exc_info()[1] - raise NamingError("invalid regex: "+str(x)) - else: - for name in self.namespace: - if regex.match(name): - result[name]=self.namespace[name] - return result - else: - # just return (a copy of) everything - return self.namespace.copy() - - def ping(self): - """A simple test method to check if the name server is running correctly.""" - pass - - -class NameServerDaemon(core.Daemon): - """Daemon that contains the Name Server.""" - def __init__(self, host=None, port=None, unixsocket=None, nathost=None, natport=None): - if Pyro4.config.DOTTEDNAMES: - raise PyroError("Name server won't start with DOTTEDNAMES enabled because of security reasons") - if host is None: - host=Pyro4.config.HOST - if port is None: - port=Pyro4.config.NS_PORT - if nathost is None: - nathost=Pyro4.config.NATHOST - if natport is None: - natport=Pyro4.config.NATPORT or None - super(NameServerDaemon, self).__init__(host, port, unixsocket, nathost=nathost, natport=natport) - self.nameserver=NameServer() - self.register(self.nameserver, constants.NAMESERVER_NAME) - self.nameserver.register(constants.NAMESERVER_NAME, self.uriFor(self.nameserver)) - log.info("nameserver daemon created") - - def close(self): - super(NameServerDaemon, self).close() - self.nameserver=None - - def __enter__(self): - if not self.nameserver: - raise PyroError("cannot reuse this object") - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.nameserver=None - return super(NameServerDaemon, self).__exit__(exc_type, exc_value, traceback) - - -class BroadcastServer(object): - REQUEST_NSURI = "GET_NSURI" if sys.platform=="cli" else b"GET_NSURI" - - def __init__(self, nsUri, bchost=None, bcport=None): - self.nsUri=nsUri - if bcport is None: - bcport=Pyro4.config.NS_BCPORT - if bchost is None: - bchost=Pyro4.config.NS_BCHOST - if ":" in nsUri.host: # ipv6 - bchost = bchost or "::" - self.sock=Pyro4.socketutil.createBroadcastSocket((bchost, bcport, 0, 0), reuseaddr=Pyro4.config.SOCK_REUSE, timeout=2.0) - else: - self.sock=Pyro4.socketutil.createBroadcastSocket((bchost, bcport), reuseaddr=Pyro4.config.SOCK_REUSE, timeout=2.0) - self._sockaddr=self.sock.getsockname() - bchost=bchost or self._sockaddr[0] - bcport=bcport or self._sockaddr[1] - if ":" in bchost: # ipv6 - self.locationStr="[%s]:%d" % (bchost, bcport) - else: - self.locationStr="%s:%d" % (bchost, bcport) - log.info("ns broadcast server created on %s", self.locationStr) - self.running=True - - def close(self): - log.debug("ns broadcast server closing") - self.running=False - try: - self.sock.shutdown(socket.SHUT_RDWR) - except (OSError, socket.error): - pass - self.sock.close() - - def getPort(self): - return self.sock.getsockname()[1] - - def fileno(self): - return self.sock.fileno() - - def runInThread(self): - """Run the broadcast server loop in its own thread. This is mainly for Jython, - which has problems with multiplexing it using select() with the Name server itself.""" - thread=Thread(target=self.__requestLoop) - thread.setDaemon(True) - thread.start() - log.debug("broadcast server loop running in own thread") - - def __requestLoop(self): - while self.running: - self.processRequest() - log.debug("broadcast server loop terminating") - - def processRequest(self): - try: - data, addr=self.sock.recvfrom(100) - if data==self.REQUEST_NSURI: - responsedata=core.URI(self.nsUri) - if responsedata.host=="0.0.0.0": - # replace INADDR_ANY address by the interface IP adress that connects to the requesting client - try: - interface_ip=socketutil.getInterfaceAddress(addr[0]) - responsedata.host=interface_ip - except socket.error: - pass - log.debug("responding to broadcast request from %s: interface %s", addr[0], responsedata.host) - responsedata = str(responsedata).encode("iso-8859-1") - self.sock.sendto(responsedata, 0, addr) - except socket.error: - pass - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.close() - - -def startNSloop(host=None, port=None, enableBroadcast=True, bchost=None, bcport=None, unixsocket=None, nathost=None, natport=None): - """utility function that starts a new Name server and enters its requestloop.""" - daemon=NameServerDaemon(host, port, unixsocket, nathost=nathost, natport=natport) - nsUri=daemon.uriFor(daemon.nameserver) - internalUri=daemon.uriFor(daemon.nameserver, nat=False) - bcserver=None - if unixsocket: - hostip="Unix domain socket" - else: - hostip=daemon.sock.getsockname()[0] - if hostip.startswith("127."): - print("Not starting broadcast server for localhost.") - log.info("Not starting NS broadcast server because NS is bound to localhost") - enableBroadcast=False - if enableBroadcast: - # Make sure to pass the internal uri to the broadcast responder. - # It is almost always useless to let it return the external uri, - # because external systems won't be able to talk to this thing anyway. - bcserver=BroadcastServer(internalUri, bchost, bcport) - print("Broadcast server running on %s" % bcserver.locationStr) - bcserver.runInThread() - print("NS running on %s (%s)" % (daemon.locationStr, hostip)) - if daemon.natLocationStr: - print("internal URI = %s" % internalUri) - print("external URI = %s" % nsUri) - else: - print("URI = %s" % nsUri) - try: - daemon.requestLoop() - finally: - daemon.close() - if bcserver is not None: - bcserver.close() - print("NS shut down.") - - -def startNS(host=None, port=None, enableBroadcast=True, bchost=None, bcport=None, unixsocket=None, nathost=None, natport=None): - """utility fuction to quickly get a Name server daemon to be used in your own event loops. - Returns (nameserverUri, nameserverDaemon, broadcastServer).""" - daemon=NameServerDaemon(host, port, unixsocket, nathost=nathost, natport=natport) - bcserver=None - nsUri=daemon.uriFor(daemon.nameserver) - if not unixsocket: - hostip=daemon.sock.getsockname()[0] - if hostip.startswith("127."): - # not starting broadcast server for localhost. - enableBroadcast=False - if enableBroadcast: - internalUri=daemon.uriFor(daemon.nameserver, nat=False) - bcserver=BroadcastServer(internalUri, bchost, bcport) - return nsUri, daemon, bcserver - - -def locateNS(host=None, port=None): - """Get a proxy for a name server somewhere in the network.""" - if host is None: - # first try localhost if we have a good chance of finding it there - if Pyro4.config.NS_HOST in ("localhost", "::1") or Pyro4.config.NS_HOST.startswith("127."): - host = Pyro4.config.NS_HOST - if ":" in host: # ipv6 - host="[%s]" % host - uristring="PYRO:%s@%s:%d" % (constants.NAMESERVER_NAME, host, port or Pyro4.config.NS_PORT) - log.debug("locating the NS: %s", uristring) - proxy=core.Proxy(uristring) - try: - proxy.ping() - log.debug("located NS") - return proxy - except PyroError: - pass - # broadcast lookup - if not port: - port=Pyro4.config.NS_BCPORT - log.debug("broadcast locate") - sock=Pyro4.socketutil.createBroadcastSocket(reuseaddr=Pyro4.config.SOCK_REUSE, timeout=0.7) - for _ in range(3): - try: - for bcaddr in Pyro4.config.parseAddressesString(Pyro4.config.BROADCAST_ADDRS): - try: - sock.sendto(BroadcastServer.REQUEST_NSURI, 0, (bcaddr, port)) - except socket.error: - x=sys.exc_info()[1] - err=getattr(x, "errno", x.args[0]) - if err not in Pyro4.socketutil.ERRNO_EADDRNOTAVAIL: # yeah, windows likes to throw these... - if err not in Pyro4.socketutil.ERRNO_EADDRINUSE: # and jython likes to throw thses... - raise - data, _=sock.recvfrom(100) - try: - sock.shutdown(socket.SHUT_RDWR) - except (OSError, socket.error): - pass - sock.close() - if sys.version_info>=(3, 0): - data=data.decode("iso-8859-1") - log.debug("located NS: %s", data) - return core.Proxy(data) - except socket.timeout: - continue - try: - sock.shutdown(socket.SHUT_RDWR) - except (OSError, socket.error): - pass - sock.close() - log.debug("broadcast locate failed, try direct connection on NS_HOST") - # broadcast failed, try PYRO directly on specific host - host=Pyro4.config.NS_HOST - port=Pyro4.config.NS_PORT - # pyro direct lookup - if not port: - port=Pyro4.config.NS_PORT - if ":" in host: - host = "[%s]" % host - if core.URI.isUnixsockLocation(host): - uristring="PYRO:%s@%s" % (constants.NAMESERVER_NAME, host) - else: - uristring="PYRO:%s@%s:%d" % (constants.NAMESERVER_NAME, host, port) - uri=core.URI(uristring) - log.debug("locating the NS: %s", uri) - proxy=core.Proxy(uri) - try: - proxy.ping() - log.debug("located NS") - return proxy - except PyroError as x: - e = NamingError("Failed to locate the nameserver") - e.__cause__=x - raise e - - -def resolve(uri): - """Resolve a 'magic' uri (PYRONAME) into the direct PYRO uri.""" - if isinstance(uri, basestring): - uri=core.URI(uri) - elif not isinstance(uri, core.URI): - raise TypeError("can only resolve Pyro URIs") - if uri.protocol=="PYRO": - return uri - log.debug("resolving %s", uri) - if uri.protocol=="PYRONAME": - nameserver=locateNS(uri.host, uri.port) - uri=nameserver.lookup(uri.object) - nameserver._pyroRelease() - return uri - else: - raise PyroError("invalid uri protocol") - - -def main(args): - from optparse import OptionParser - parser=OptionParser() - parser.add_option("-n", "--host", dest="host", help="hostname to bind server on") - parser.add_option("-p", "--port", dest="port", type="int", help="port to bind server on (0=random)") - parser.add_option("-u", "--unixsocket", help="Unix domain socket name to bind server on") - parser.add_option("", "--bchost", dest="bchost", help="hostname to bind broadcast server on (default is \"\")") - parser.add_option("", "--bcport", dest="bcport", type="int", - help="port to bind broadcast server on (0=random)") - parser.add_option("", "--nathost", dest="nathost", help="external hostname in case of NAT") - parser.add_option("", "--natport", dest="natport", type="int", help="external port in case of NAT") - parser.add_option("-x", "--nobc", dest="enablebc", action="store_false", default=True, - help="don't start a broadcast server") - options, args = parser.parse_args(args) - startNSloop(options.host, options.port, enableBroadcast=options.enablebc, - bchost=options.bchost, bcport=options.bcport, unixsocket=options.unixsocket, - nathost=options.nathost, natport=options.natport) - -if __name__=="__main__": - main(sys.argv[1:]) diff --git a/contrib/site-packages/Pyro4/naming.pyo b/contrib/site-packages/Pyro4/naming.pyo deleted file mode 100644 index ed929bb8bfba86c5e0f617866ba599dd4829712b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16892 zcmd5@TW}m#T0T9a+l+OyY|ECN*lov3#z{0zve{g`Nj9>rBu<>M(<4WYlI-kgx<_hh zrhC+<$CgqSc$4fx6}4mw9=PU#q9_WArJ#TZDm<{yTacx8Yae*wfh`K2sDcL`xPIS% zx~E6BLoF{BlKSZMxnKUv_y6Z~{$Iz6U;F#tavCc6Dd7Lhx0TYrX({C@wT#eGZceo< z)y}EqoU-_ySFOBS&PyKR18R97jSs5j!8AUkmWL!hpjyMKJ))LJRJ)*-3#wgI%SF@H zplahKrJ6sfl}UpT0f-RQT36fmZz0Fro6+diSduCkCpmJsdtu-D0f161+_jS z8I!$?qsm3YrubAZ^%&=lKkvaLVqO^@P#|%X8#xvIs^#GovpcQa8Fd2FurMw3RxKUFPU$X!Vx`{p?5Y>t@FKe&xb~XY z>LA+e294Mc16?i_uWm=7J!fCtj@QD#cD!~Nd-j!;b*~ZYvTa`oJKK@Jx)$3j+xDfX z?FF{$*{_Ges$KH=`f^nNAdGJKfmaT^_)NLz|2KY0pb-_1Be?0uYc(C$V-JPnN-56_ z(ozHc$MqoA@qoNVUR*>(hm8#{?#6zrlw;9B1kPn7#zTlLu0>wmH92gAqhE-kFpBr# z4b%05Rr5ZA_g2_Izm;k{jF-sMVe5t`adaHTm8#Fd_;{2aMZm_ME(g+xjD^c*RD7XE$5**fwQ4^@eB1q1~=`>{hty zH|i}rl%13Unt9UXHRqBpH@HQ7xXc%BqstjYuo}0Ns$t}=SH&vH{*XD|CJQ9ar|`$~j%Zidl@K5Iy@+)Pu> zzU@VtRC@NQ^0Q}K{>s^o5GmiZAA57?eWSkWu_iX|EJlRgFQX6_uvYT}Kd#j{Db4XH zt6()aS3aaFne)qtY#&jvQtK9HzMfOJEHwjmsxAd5)j*tA8$}g;MOl`GNn)3geHhsv zS}L&Ab?kOkXj+%i(&aGR=yvRQ&9hhi8{ieL(&IKryBh`Aw7|yF+kVq-hTXu`g20l=4i+bCqdKB7LEQWlyuWs(2F61erP z>jVXhUWzV(a zn|{M97wzO{5hT-%e3oUwx~<~~)NF?o+(I>mlgs*H3?d9#+f3BVHzjpiHS0r9p{R_Y z-PO{+G%I{A$@Lbc>2}|+NP(2ABK~~si--n~lOUZ*W^gC#x+csIxjAPq(NH{A0WBtFViyX zAn5^lfv^FUf|!-HK1kjGx$g|5)IX3A^DW?*p_G>l2G}~Hkb3d`NwqU5Euq6f_2TBQ zsU?s;vd~b~bf2)uOENBua98gb26^&}d!Mxa54|6@?$x_H2;SnF>aQRm`nW;jH#-v5 z2~x^KGsxiF(p43kdUJhwmW{y6C(En zbdy>%ScFp4+iC!EHA4~k!YP=5lx^4J#+ooh1J*mb=>l{bl zm+^?ERRcvZ$q<-l0*HnTG-Tzi!+4*xj^<42aK6FurjM` zI5bH_%V}TFoITItN&TGV_&sOMA{yzH;R% zVc>}fbGRAK69}@2O6-2x6AmXvTp)lQ6p$+n6oyA9r;bb-tf(Fv$ zhh%;JHY&~|;`LjLfVBO8FdpF@fyy_=qp=)O9+o2vMnr~b7_vD%i@9?o-8kdYyg;H=|E6N$0&C*U=1-Zw(Q1;ik?Y}4XPajq{Rqi)>~=T zkbrg6!l;3Jkkv}FhN%vSEiez0_3bokM0I|Si4fl?0ND*1fqDUYhrGoAh`+^#X0y;6 zknH)K6~j-1=0H70I**HthAA+i)+bq60L++LA5W5~vGym0pc+^ro>J=vxSV*mFam?= zE1Ik>%a*X+%`kW*hVX?1v<2K>yt25sa8ba5?FIFf79{vrT_hJR5yuaJZ$;msxxO z!E0Bli_%S{TDyA1N#gU1S^E5vscx%lLDgXOZqahaltQ`^v(cE(lI(3KV6HLY(h@ z4p}9nOk1a{o-pTpFWg2Q8ximgG-*PrMCC$6BuSzF)IY*ic3x({RhA8Hv_i->ZkR(c zARdxEl-SToO+KPiXd-h{ka}eSC}bfcxV`W$z*zxNWTX^u zYAU5ZqO%nu6>c%v_TCov5x@wt(d`ypQJD$c157cjWf>aL6L4AAyu}e5Yb$VX?Gd-U zj#2+So}S~ZV(h0Uk>)(b;7bf-!XijWQ_dv@teTBlV)vEN$#yL~nqrM;?Z|#;>p^G= zsNyKByN$Yzjk0baNq)%a=WnCV2Z(s0rakdUfduGlhie~NP?RNg^gj86riPOq3K~9M zQP7E|hEW2iE>ilOx^VU~1r}KD3NjMcOr3-vW=E6RPpM|^!y=TbSt^bq)+e~u>~+lG zfk|1s{am@KVAqP6O6LU6o;wF47rR01B(kmUx{>JF*l&AbH(q5neqJ_~75C-cS^fUG zSyxjS%xcC?A_io(lG0rY?GvSM`G%{W$E$M=fs6pU{rm;d;vA2> zQNKG2EJf;#4u z{qsuE1t)H{m_!M0j~9e{zGD=Ce~hPpHXJC$k<3Psc{5P(G24Y3^Hsu zs2Yk0%7@7i{#6kv`ghTo(={%#yQC-%ojNxBXMPOOi6gXF4u|Pu0}91o8)h-iNt*D6 zy)Gw*kM}%hH`n~en!N^BT_*}*(ZQ1}wqBb~I2@~P(d**Jc9*;dsojG2rF6!Wp`WJT zU_|#g>MRS#!D^&TeGJ3C_i$5>4u+i#xl$VDJgEX=8eO$ahReBctrmHkUAT>KzSt3V z)?RFU01nTIMI?yDYUFesX?^E|BpS0edn1BH)nMY60&{%-rx=epCd4_*7*AHn0epHK z&IQ~u(iQr5Mz}&)-P+9K7(*aPB8^3TvtmWm?Y);Bs2_*JI8T5ir2-SFDfbhw{|!%T z$1HH)`f)saXr{#!2D>)?FOZNJEu(PW%c7BF_z1)O6C&Gxok7YK8iu)KVODrSOYqbHKe9=&tSHJS1jgbOLI6e4&Qfr&zl)!wI#jlt zR9*2ARhr!ML|J|XwP_kAa3^=dz$O2}%bJlz!b)~C3fn+m=O95q9ZlYynAKu6_&S_> z?zt~NKlfyb-n~q@!UubU)nsp+g}1IPR1H}tUaY7_qN;{bcjwHP`*s|!~ z3yhIv9a6oB5ie+ju4i<(7r1d~3{2`q!C!727ivhyg4Jj?nAvJia+uJYRn7104cj}| zLG?X6iO+>s0w#dACUSY;tT8yyhpnU5A?rcJgqX>3e1w=eJLtkx1>$Yb^|%CkMSf^s+7c>&+Pm&$P-}M z!962r(zdgSlshf|Cev0Kv>A{IW%vJoxCAN~&bRUCg9wJKW4Vb#g+d{Z-;u&#VXBaC zj39&Zmj8wNc(M=7kG-VSgrSR2;K?lh|5rIW$(HQQK zkkR+Wa}I+R0uvAlS@14k5D>ll)by*bF(O`bxfY0WXbGbjr9r}MypE~p-|5P@Rb&e8 zLf$Gc5}-qfX^DRTPdiGE3r1VQO!zh-Z|DH;8wC|TXR$Mr3ZB1?Ym;bh9K1{MsYznI z&qp75dh-xi4M!IOS$J9+csF-JD&AETR~>L1z+ED;iTBExCs{lt(CKo`Y}#kH6x<0hvriAKLvqe{%e}~p<<3sdTZgpZKa&AGkBFj?|?wGo;e&b&V+r)PFDf`5s^kK z4YO%dnESO?N=~LPG*zjRKK~Yj)wIi1?l^Sa5jij}z_T!64T@DeG%y8U!x-`k^hM-o z=Z@!!NCTw1OAh@$8v5Vm5adN7d*GD-a|*hG)PcOwpGu0jCn#p%=jY0c7BGrw;3!l z_yz+W#FX%u;J!M>is<#f@aSpwpFa*#ox%fB1yRCpj^76DDx%2(9#M=x&Wl#Wg>;_y z{$maOgNovIJSACMhS)a3CiWhT9nWj|w&iqiCzc<7H?~jN{__D;7?-x6fN! z-%U#&NFxVLiC>hq{Wh2KSP4X45OFwF1A)M z9#V+9C};7h$!;o7DO+kMlng5C5jq^j8KL9lry=y9+#~eA$y=ZkD}Iz8ldv)r0N7`r zsy)Uqy~TbQ!uu$ATvH9dDiRuwkuBpq13u%mk_0B{5Eu5>JX}zNl$JY7Ud#*pWpPYp z$OnQ#ojdoW93c{cId8Fj;xV3+2tv8~j&tSQtbW-Xme5jWb-4v6PPXc~)FhX5D27T^ z6H{i`&15(Z5}hi7)u#OX!&kF!JIu{cdP#6;EeMUvfWj}c^Tq3&^&#Cpj}mwGV>jz? zzPgX&cK0?e>hVq%@81OY4>ZK%9GuPYWJdf5+4S@YH1>+rP2nNFvPmy|nT4$wj~y6C z>16FBC7Vcz81~NNA*$S@Q-j^a)O?U7Uu1BBL7f3*xp)fk5y{rJT&gm6Nv<)MT{=q) zuBYXqdH`j85U-tjq`ky*;iZ?qMTo=PVCQk0uW2yQP`I2xS#xcXdahQJyFECsXM24pmYKK; zlU#NuK?#o-{}7kGCLOLc=#_EKB0)?PP@U7oi^J;@5HX;`J<|9i=z9VK(BDEtIPha=0F(Owykqj6X*NWCWRswSw%!=OM=5Yvd}S z0#Zx?K^`ch&(kkv{GG{{AL7@;Zz*VtMWn?w6r{Ed{UNQ3JW@|;C}PY^CrU_v7l zshgZuKw=_Az7#~~P|!R;hI*4WQ#U4hgUwOc>$IsTX54VJASXjPUGugCfXU#R-v;%Q=q<4MM zEDt*3uo1Qr7Z2ImA@b{UT^`0OG>_1=l-Vy<_mtIed)tLli(O!)uh~+>m=>)ih?8F3KD@_%%V&Jo65No zv*jRRR}*&Wi5H!asqPvR6{tBJo0~H?D8!x=m%?2$ z$>2$EPIk%lnsuCF(jRr|)EPNvV^cD{ud{KUGVGeoJ)6nqMgEzaOK!A^pGMnA>banL z!vY>`fc1GPwlcq{d-bI2=TJpI&|go+a7P{4k*#D$f^elF;>fs;@Is+d7p#@|UBbw# zHFdJW-f$SKr z6R$#f8K6i;aR|H3^<&b4x!gp^FV~{W<`|paCHd^w9GO!-m~8nfMH^I0U@U#qYV}qN zJ^v!xBk_yc6Ke(V_A*9*JJA=I{u>PTArL}K4%o$KT49bjh2CdOcAnJWh*~72JAc67 zNd&T-Adnl`Zc6NaI(KA)UN9eCyhNlbq&0*OR-V9CPUCkpHvv^x$W7#@-<~-+{pQKj OC-+ZW#%ajZsP#Ymiq4w= diff --git a/contrib/site-packages/Pyro4/nsc.py b/contrib/site-packages/Pyro4/nsc.py deleted file mode 100644 index 8d063520..00000000 --- a/contrib/site-packages/Pyro4/nsc.py +++ /dev/null @@ -1,102 +0,0 @@ -""" -Name server control tool. - -Pyro - Python Remote Objects. Copyright by Irmen de Jong (irmen@razorvine.net). -""" - -import sys -from Pyro4 import naming, errors - -if sys.version_info<(3, 0): - input=raw_input - - -def handleCommand(nameserver, options, args): - def printListResult(resultdict, title=""): - print("--------START LIST %s" % title) - for name, uri in sorted(resultdict.items()): - print("%s --> %s" % (name, uri)) - print("--------END LIST %s" % title) - - def cmd_ping(): - nameserver.ping() - print("Name server ping ok.") - - def cmd_listprefix(): - if len(args)==1: - printListResult(nameserver.list()) - else: - printListResult(nameserver.list(prefix=args[1]), "- prefix '%s'" % args[1]) - - def cmd_listregex(): - if len(args)!=2: - raise SystemExit("requires one argument: pattern") - printListResult(nameserver.list(regex=args[1]), "- regex '%s'" % args[1]) - - def cmd_register(): - if len(args)!=3: - raise SystemExit("requires two arguments: name uri") - nameserver.register(args[1], args[2], safe=True) - print("Registered %s" % args[1]) - - def cmd_remove(): - count=nameserver.remove(args[1]) - if count>0: - print("Removed %s" % args[1]) - else: - print("Nothing removed") - - def cmd_removeregex(): - if len(args)!=2: - raise SystemExit("requires one argument: pattern") - sure=input("Potentially removing lots of items from the Name server. Are you sure (y/n)?").strip() - if sure in ('y', 'Y'): - count=nameserver.remove(regex=args[1]) - print("%d items removed." % count) - - commands={ - "ping": cmd_ping, - "list": cmd_listprefix, - "listmatching": cmd_listregex, - "register": cmd_register, - "remove": cmd_remove, - "removematching": cmd_removeregex - } - try: - commands[args[0]]() - except Exception: - xt,xv,tb=sys.exc_info() - print("Error: %s - %s" % (xt.__name__,xv)) - - -def main(args): - from optparse import OptionParser - usage = "usage: %prog [options] command [arguments]\nCommand is one of: " \ - "register remove removematching list listmatching ping" - parser = OptionParser(usage=usage) - parser.add_option("-n", "--host", dest="host", help="hostname of the NS") - parser.add_option("-p", "--port", dest="port", type="int", - help="port of the NS (or bc-port if host isn't specified)") - parser.add_option("-u","--unixsocket", help="Unix domain socket name of the NS") - parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="verbose output") - options, args = parser.parse_args(args) - if not args or args[0] not in ("register", "remove", "removematching", "list", "listmatching", "ping"): - parser.error("invalid or missing command") - if options.verbose: - print("Locating name server...") - if options.unixsocket: - options.host="./u:"+options.unixsocket - try: - nameserver=naming.locateNS(options.host, options.port) - except errors.PyroError: - x=sys.exc_info()[1] - print("Failed to locate the name server: %s" % x) - return - if options.verbose: - print("Name server found: %s" % nameserver._pyroUri) - handleCommand(nameserver, options, args) - if options.verbose: - print("Done.") - -if __name__=="__main__": - main(sys.argv[1:]) diff --git a/contrib/site-packages/Pyro4/nsc.pyo b/contrib/site-packages/Pyro4/nsc.pyo deleted file mode 100644 index cb3878bbf6e1d62a5a1e326b36e7ee5d2652444b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5439 zcmc&&-EJGl6+S~!lt{^zBH3~jJMq|wV!8lHv<=)qC{hQu6DJTDVdd1R6PKX4Lun;) zmzf#LVjl0gw2=P9nx%~M5+dlYE9HqY}9kW^cpfzi&` zLt}L}OTEqd+Gkn6QLS!`ZKmsbYwY*3R7Ylzd85DD**9J98d^WdhGU!T?s>g4)?e6x zNp)=Wmsz^2YY7+c+s^MYdzhrAks5!uQBD4XU#nKYyuOSi?F^E1SF*<1%({fJN$^mY zbC>YU;NiUcH*S>-lf{$*0j1z4^uV6TR+S1vTqV*Xy5fY2{qIN3DmGR%gfT zGV4w3`6Lr>2Cl~A`zjJ^+^Fy4r0b>DC%$hy*FrOl=VKq)q=pws;;=euW<%fXXWdS} zxtZ+PjvY5YvuOSzvj?sjl(o^DNSePkP^&C;&7U=X(d;KX&7r6>mmF48A9lJ2on6B{ zZT3+!b=?M9EMlUeO;W#^I3LL>S>)!BkgBMgYDIN9&p(BSK@cQf0d+tQDURdw$5`>U zAklJH(B%AW8$$PWcF?GCf7vxl!fr*x2r&@hc3v<7ywmO=Zu6R06C584sp{Fb`8;LY zEo7i?*f}%}yJqfLr&k4f1qGjIpi0k)pn+(Ornndix|bJLPYWnAVmw(Z5fqpxhSv0w zBgcGD*P)>Ath+lw$5>DLCKU+#IIkMYRAZ@#I(FAZOn4FNtyEr|B3;Im3?r{AEPM?T z$D1OxtX2wBEtR;0RFvV-p|nmi3RjZ0m>PH5`{A;d8$^zgyG&p zV$JuX1ouv7snLLD1fupEdf4&a*i^2eHM{0Wz;_GiQP&~}fHSHQZ%Xd(_SoShQ{>%-<}0qjbAq@Wl&G!i^o(eHWOkuE#+n#xD>p+9AmY2{ z=Vai}K)OA#BU6Krb-CVe8E|xghoC_h8aVc zM7TlGiI_1c{y~XsbE53v4YrFgz@-Vr^6d$PA?|}??qrYUP*8KMovqC8G3DAI4z$a+ zBM3s7$LVIH)W2Eyc=-W7-oj5%%E4Pv3u-Bl#>58@@C=&#r~-ST3vdYBi&J<$0o)25 zl*x7kd$=}1jc0{VPCz{oj1zHqm+ueq-vBPQO16;PWZ;GY<~xY4Hej z&t?Pd_l!PgARGEqYxFo9X*WWqHg2YOe+3#O8cX_{Xc_syQ|ub8$N6CS0XM=C-au9K zE|MCb?YXPN#Uve$g74wHO@HCw?F3^&qnjnQP&JT+$izM z99W-u#~gLrN!rV#&~7u2wc9l&UtxaF{0C=^y{~{BuPWlBimq_-HYe9PxrIbV>1KmL zCygC6mFUQa*P*=pPM|fhK(CvZi=#E(+#VL#HxI%}88!DI5>gB0#nRRC)tTk;^6c_l zQO!>Os)Zu_#Y^g{>azMWE{y5}Jnqksfb2kVeWC#v4^bNI3p{l=hO`^wZmUW zv;cd98yX|pK%`OOK;*PgPlQixN{JU9HItXm7&dc^|!g2KSu~H3ar1KxK{3YHCGj#1g9(OZGZ-Y zSqyw=^4H`-YQDOXOeO4wi*qLPI=&wDR9u>ReJ^uf_$oGeW-8e;{h?!;WQjN$pj_eE z?J&kLA7h9ZO~#mJ{CF5bFhoJ_=NN)(n6X25UCXTA>DEP>^fXU^9j14@c0vepf*vXucuLYg1kP-8p z^O-em53fh~CU52VUgmk7jXYRhSniAmqxaE5yw>P8=VUDA`XoK<^pjX)dxOL|wp9~W zx`ygaM4q07f@TQK8jXg8M~&uaBiJK$cRuSReRv-}gRMg!Bfy>O#XKuPc8eXr{1Z$M zzDF!IY@Qy%NpIBHM-lTs(Hc3kz~Qpd*XdlA!Wty7pSJ>1BS9vUun;a3cgRW|HCVzR zbmF)j)LDvB+vXTb)K9+CMC__b?b(q=Y?2~8Vcf9g)^^026EW+FA>klfj7+r7awg7T zJ@6?h``RW^9g-RoWUv^#s;J4W7zA01jo1zZ3&Z0c1+#!0U7Q^#VK!FW$2{vtJhxg> z<-(%6p{|4WSJiEh{WZucY7WneDxzjd$}=c0N-pfY&EvG&qBr7toP7kVG4qI8x7%^n zZMOwAIS2V#A#ew9xh3?SKXn9~%m&7GUZ4Qok2vxZY$G;0(H$gm+i|XkXZT6(;y|3k fDt~f|1iuy`(MZT%jd_c%6{KW{)JS?1$CA^3uoD5u{U{1kS}f3+6}^*7-EAkT6VMz z5RgX{D-9S_o-Y#<%wd^gLmpPxQ$%-(L;@JOMU}|o8*G+ zJMri|Dv$9*<@d#uEglMtsi`uikT&~A7)5FtqVA}&oxXD=TTc3#ZI5rnZP2?kYW?Tv zC*~Vw{N$Zfa1rOj?$~LP=}ZTzT8z6(wK2}SXpEAXBUydXQ@!o*@<93^ Gm3;%avSt(j diff --git a/contrib/site-packages/Pyro4/socketserver/multiplexserver.py b/contrib/site-packages/Pyro4/socketserver/multiplexserver.py deleted file mode 100644 index 20730187..00000000 --- a/contrib/site-packages/Pyro4/socketserver/multiplexserver.py +++ /dev/null @@ -1,223 +0,0 @@ -""" -Socket server based on socket multiplexing. Doesn't use threads. - -Pyro - Python Remote Objects. Copyright by Irmen de Jong (irmen@razorvine.net). -""" - -import socket, select, sys, logging, os -from Pyro4 import socketutil, errors, util -import Pyro4 - -log=logging.getLogger("Pyro4.socketserver.multiplexed") - - -class MultiplexedSocketServerBase(object): - """base class for multiplexed transport server for socket connections""" - def init(self, daemon, host, port, unixsocket=None): - log.info("starting multiplexed socketserver") - self.sock=None - bind_location=unixsocket if unixsocket else (host, port) - self.sock=socketutil.createSocket(bind=bind_location, reuseaddr=Pyro4.config.SOCK_REUSE, timeout=Pyro4.config.COMMTIMEOUT, noinherit=True) - self.clients=set() - self.daemon=daemon - sockaddr=self.sock.getsockname() - if sockaddr[0].startswith("127."): - if host is None or host.lower()!="localhost" and not host.startswith("127."): - log.warning("weird DNS setup: %s resolves to localhost (127.x.x.x)", host) - if unixsocket: - self.locationStr="./u:"+unixsocket - else: - host=host or sockaddr[0] - port=port or sockaddr[1] - if ":" in host: # ipv6 - self.locationStr="[%s]:%d" % (host, port) - else: - self.locationStr="%s:%d" % (host, port) - - def __repr__(self): - return "<%s on %s, %d connections>" % (self.__class__.__name__, self.locationStr, len(self.clients)) - - def __del__(self): - if self.sock is not None: - self.sock.close() - self.sock=None - - def events(self, eventsockets): - """used for external event loops: handle events that occur on one of the sockets of this server""" - for s in eventsockets: - if s is self.sock: - # server socket, means new connection - conn=self._handleConnection(self.sock) - if conn: - self.clients.add(conn) - else: - # must be client socket, means remote call - active = self.handleRequest(s) - if not active: - s.close() - self.clients.discard(s) - - def _handleConnection(self, sock): - try: - if sock is None: - return - csock, caddr = sock.accept() - if Pyro4.config.COMMTIMEOUT: - csock.settimeout(Pyro4.config.COMMTIMEOUT) - except socket.error: - x=sys.exc_info()[1] - err=getattr(x, "errno", x.args[0]) - if err in socketutil.ERRNO_RETRIES: - # just ignore this error for now and continue - log.warning("accept() failed errno=%d, shouldn't happen", err) - return None - if err in socketutil.ERRNO_BADF or err in socketutil.ERRNO_ENOTSOCK: - # our server socket got destroyed - raise errors.ConnectionClosedError("server socket closed") - raise - try: - conn=socketutil.SocketConnection(csock) - if self.daemon._handshake(conn): - return conn - except: # catch all errors, otherwise the event loop could terminate - ex_t, ex_v, ex_tb = sys.exc_info() - tb = util.formatTraceback(ex_t, ex_v, ex_tb) - log.warning("error during connect/handshake: %s; %s", ex_v, "\n".join(tb)) - try: - csock.shutdown(socket.SHUT_RDWR) - except (OSError, socket.error): - pass - csock.close() - return None - - def close(self): - log.debug("closing socketserver") - if self.sock: - sockname=None - try: - sockname=self.sock.getsockname() - except socket.error: - pass - self.sock.close() - if type(sockname) is str: - # it was a Unix domain socket, remove it from the filesystem - if os.path.exists(sockname): - os.remove(sockname) - self.sock=None - for c in self.clients: - try: - c.close() - except Exception: - pass - self.clients=set() - - @property - def sockets(self): - socks=[self.sock] - socks.extend(self.clients) - return socks - - def wakeup(self): - """bit of a hack to trigger a blocking server to get out of the loop, useful at clean shutdowns""" - socketutil.triggerSocket(self.sock) - - def handleRequest(self, conn): - """Handles a single connection request event and returns if the connection is still active""" - try: - self.daemon.handleRequest(conn) - return True - except (socket.error, errors.ConnectionClosedError, errors.SecurityError): - # client went away or caused a security error. - # close the connection silently. - return False - except: - # other error occurred, close the connection, but also log a warning - ex_t, ex_v, ex_tb = sys.exc_info() - tb = util.formatTraceback(ex_t, ex_v, ex_tb) - log.warning("error during handleRequest: %s; %s", ex_v, "\n".join(tb)) - return False - - -class SocketServer_Poll(MultiplexedSocketServerBase): - """transport server for socket connections, poll loop multiplex version.""" - - def loop(self, loopCondition=lambda: True): - log.debug("enter poll-based requestloop") - poll=select.poll() - try: - fileno2connection={} # map fd to original connection object - rlist=list(self.clients)+[self.sock] - for r in rlist: - poll.register(r.fileno(), select.POLLIN | select.POLLPRI) - fileno2connection[r.fileno()]=r - while loopCondition(): - polls=poll.poll(1000*Pyro4.config.POLLTIMEOUT) - for (fd, mask) in polls: - conn=fileno2connection[fd] - if conn is self.sock: - conn=self._handleConnection(self.sock) - if conn: - poll.register(conn.fileno(), select.POLLIN | select.POLLPRI) - fileno2connection[conn.fileno()]=conn - self.clients.add(conn) - else: - active = self.handleRequest(conn) - if not active: - try: - fn=conn.fileno() - except socket.error: - pass - else: - conn.close() - self.clients.discard(conn) - if fn in fileno2connection: - poll.unregister(fn) - del fileno2connection[fn] - except KeyboardInterrupt: - log.debug("stopping on break signal") - pass - finally: - if hasattr(poll, "close"): - poll.close() - log.debug("exit poll-based requestloop") - - -class SocketServer_Select(MultiplexedSocketServerBase): - """transport server for socket connections, select loop version.""" - - def loop(self, loopCondition=lambda: True): - log.debug("entering select-based requestloop") - while loopCondition(): - try: - rlist=list(self.clients) - rlist.append(self.sock) - try: - rlist, _, _=select.select(rlist, [], [], Pyro4.config.POLLTIMEOUT) - except select.error: - if loopCondition(): - raise - else: - # swallow the select error if the loopcondition is no longer true, and exit loop - # this can occur if we are shutting down and the socket is no longer valid - break - if self.sock in rlist: - try: - rlist.remove(self.sock) - except ValueError: - pass # this can occur when closing down, even when we just tested for presence in the list - conn=self._handleConnection(self.sock) - if conn: - self.clients.add(conn) - for conn in rlist: - # no need to remove conn from rlist, because no more processing is done after this - if conn in self.clients: - active = self.handleRequest(conn) - if not active: - conn.close() - self.clients.discard(conn) - except socket.timeout: - pass # just continue the loop on a timeout - except KeyboardInterrupt: - log.debug("stopping on break signal") - break - log.debug("exit select-based requestloop") diff --git a/contrib/site-packages/Pyro4/socketserver/multiplexserver.pyo b/contrib/site-packages/Pyro4/socketserver/multiplexserver.pyo deleted file mode 100644 index 048c60f11570ea720f3816024a251ddf75f4073a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9764 zcmds7&vP6{74F$tt+cXa$-m^-am*yd$vUy6{34-@<4PFMbouiyLL_ukn3=jia;fB&`LQ2CFGzpvpkzef=&wSdx5VL`PVwSbDP7t}(bS9jHd z+p8DVLa|pLPzwW6cU7yT)&|wWpjsPJ3qxwHtQN}3$;T?Gg_6=m)f`q~QQgNh`&2lf z^oUx*WF_^!Qumem{=%pV2bCU`)}emueifE`t;7A+F|N2;*~g9i9K~=oX{_qZGdjJi zQ*SXaI`op*Gq!oH)5@ZDOK(K+@}xJDXcIr3c^#v@Y$eq}XeNh;Z*Hb={LED-@NAe1m9=!TsFjs)tJ5?|O;$veZRR8@T*FI~wxiw3WPj;8%pOITYrU=L z(C)b^TfT@L_s_V1eiw0>6%-n03^}93iQ)(x^*)XOr&ypgd4#T1N<1C54DxhX8Ol!w zXQ}bxARF`Jlk-^^;n}d~sPgl};Ix#!*JuUCcuPs@?bv~rr9o`kN!mXtcE}H@k;E~C z9VM}83}Bc+TxDF-xXhalq^ML=G!%<)LZvp9s=I2lsM04`#$~lxP@Aq=DyXdi1<5wC zIkdSZ|KAYdEA<-TZ04^9Xu#7Y8UfbiH3t&ibb+#-!NAXFMNNj&cPD zReY0O+F$gh+>*_;#~dzUpE|Y$tHe4BYPF=&mz)qw?QMS|AK{P+_dnE%wN@)*5bB(H zoTANwG=p92I;Wi~C@Q)b#bL(NNHxqR2*Whv`DD?WPCA)9iIXT^(P@-Lc%V!X#q%$` zGAYkmNh4^jBqlT8Li0TxrJ*-7SB1T1o%T8Jl<`t+lGa^qye#qhy}b%Y*xq1 zi)oN{r`|}h+IYrry%2q18^kF!{<UExBz3vm2t-k5lKZ2VI*W#aG4cDHLERVciPjx%3*H58txO?UFrN>*5 z-WgP^=t2NnSwTVUEo68fZ=ec|9 zm<+wLf_{xU4N8Ki?&1=OypPJp7QMpEosJ?D<5!dmxSw8;dQaWp6@blfMCqoq!zIWq z+MV@HwLU1VMGnEPs1y~D47YGt6fXzH4V;biaPS6xYtUz_u;w;a0~ij)zv_)lr*Y8o z^j*LWz%yx^bKXi2hb?W}0IDlN<|U0rC#8-91ifSljXL+e(0txTCMQsHf-!QG*#XQ| zx1&w>J+x@Ks6609z{U3P^?FB}%zqj$2?mCdX#{B~CL{(Scq=Q-aX^sV@*omO=6-J%0!a*R>ggb_E*y;^^1651(<`+PM?j`CCEeWaumkfi! zJ*W)M`Zl-LSdE>7tJE)pi zJGwjSJ%@(>9?aiSx4NfgL814f?Qh4veL=IkcIRKp8_=p`ZqE9LYW+dZGO&btL2bM!Gw+8Vt`IuA z65t@T?sxG=I?NcPol6)y#&=d1obO|NgaghO@OtJqisL# zhpe(uIJ~BD84m^Z3-AOw1QsRE(P|u#9rq70%v_wAjzi11Ihqw1O`QTWdr3|LP`Qew zTp-m!6-FSK1`m~2Um(7iu4|w zYNx${nhkCWLgo3SPsx6Qn=aUfXdGp1p#}yEjb<)^)k!ABc`jX2dt%w5_E-itykb@o<*+tlQ^N_iU z&<$o5{AXD(p|S(;sI2}46rvwhjj$rhy4Ky(`rZgyM*FX@*SA@SZNAQ0pI7)7aa-ZJ zSSL;I>tA8fGs^G$Shn#P?MaL~zmpWG=Kk^BMLY(ULwIuAc zkRuTLN3fYRkS}0#>_5TcDHhzL|2zx&bL&oqP-~}28>yhKj~u=7Z?oWOY>Px{Uw_8N zcd;sNT;x(hBs6lQ?3M@0W97%oN5)F!q6Co^K&paD^&3g6)xazAL;avWn8#3P^hQGCTr@)@sPWW<~h&H6{q`o}Wvc;19C6hWu?Saqevj=QxF zu{hc80o6Pxo*{xq;M=!2_91Rn)(9yfPPJoi?&I7{)!TI*QE1_$T+u%Ds+}9t*Vf-t zwOV`<<4?%of8pTy5xVjUd37y>^p7&=MidXpr~i`e+sP4~-^$;R^OX2r(00z8fMy_# z0j@yHILnVU^dsU~$PA)?xKO4?(92{=yGf>Gy?gk#7l_2<3y@Au3x=+c}|d&)f@ih z?bsOVgseSU^NhTO(r3_UJ=kux?w`Q}LFi80pN*XVP2F8gkgvK-0i~U`P2{fxhEa1x zaGUZIzeiyqqz&=G1FRjer3B{ef?wEn#S%iK+^3|vQn{z>4eSOR3`GuWfmxLp)PIg6 z$&+{+b-sR-x2J_Y3)o!3BFXkg99OwO{9~wl2q;HTJA_OaL(LLe#*v+8Zj8PX5kX-X zD{W9ym%=9$7B1_{2mz#&)40q};f{CtvW@kgXZexS zouX?aqzg#GyV2L|aB5EXSxF8`T74P8CI*u)!pxu+cMy3O@cty=7hjBr1%ZD|;-9;) z5198vPU1!{xdDe~Kf!*c@~k9~9;>jxTZ<21q!AhB4?A-r_QSGtvR?r3`PVKU(E@`9 z`lHX7`q=O}!_1v4`KJQgi=O0jc?0fCy4x(TnRak%zd?%=LNs%rKj5^p{<*?$t+qan zbxBT_q0QU$G17Ecmb%INT+}xHyn&~;aMgBsA~+!%yV!5Z;#X|!Nu+<+x%^kZVoiiD z#p?CLuUD96+;%Gd5^vci{z0;_q0Q4^VQSku_^Z5?WX(K#GYokvXmzy2EdS{%n9>jr zVM89OpudJ" % (self.__class__.__name__, self.locationStr, - self.jobqueue.workercount, self.jobqueue.jobcount) - - def loop(self, loopCondition=lambda: True): - log.debug("threadpool server requestloop") - while (self.sock is not None) and loopCondition(): - try: - self.events([self.sock]) - except socket.error: - x=sys.exc_info()[1] - err=getattr(x, "errno", x.args[0]) - if not loopCondition(): - # swallow the socket error if loop terminates anyway - # this can occur if we are asked to shutdown, socket can be invalid then - break - if err in socketutil.ERRNO_RETRIES: - continue - else: - raise - except KeyboardInterrupt: - log.debug("stopping on break signal") - break - log.debug("threadpool server exits requestloop") - - def events(self, eventsockets): - """used for external event loops: handle events that occur on one of the sockets of this server""" - # we only react on events on our own server socket. - # all other (client) sockets are owned by their individual threads. - assert self.sock in eventsockets - try: - csock, caddr=self.sock.accept() - log.debug("connected %s", caddr) - if Pyro4.config.COMMTIMEOUT: - csock.settimeout(Pyro4.config.COMMTIMEOUT) - self.jobqueue.process(ClientConnectionJob(csock, caddr, self.daemon)) - except socket.timeout: - pass # just continue the loop on a timeout on accept - - def close(self, joinWorkers=True): - log.debug("closing threadpool server") - if self.sock: - sockname=None - try: - sockname=self.sock.getsockname() - except socket.error: - pass - try: - self.sock.close() - if type(sockname) is str: - # it was a Unix domain socket, remove it from the filesystem - if os.path.exists(sockname): - os.remove(sockname) - except Exception: - pass - self.sock=None - self.jobqueue.close() - for worker in self.jobqueue.busy.copy(): - if worker.job is not None: - worker.job.interrupt() # try to break all busy workers - if joinWorkers: - self.jobqueue.drain() - - @property - def sockets(self): - # the server socket is all we care about, all client sockets are running in their own threads - return [self.sock] - - def wakeup(self): - interruptSocket(self._socketaddr) - - -def interruptSocket(address): - """bit of a hack to trigger a blocking server to get out of the loop, useful at clean shutdowns""" - try: - sock=socketutil.createSocket(connect=address, keepalive=False, timeout=None) - socketutil.triggerSocket(sock) - try: - sock.shutdown(socket.SHUT_RDWR) - except (OSError, socket.error): - pass - sock.close() - except socket.error: - pass diff --git a/contrib/site-packages/Pyro4/socketserver/threadpoolserver.pyo b/contrib/site-packages/Pyro4/socketserver/threadpoolserver.pyo deleted file mode 100644 index 3455e45ff5042cbc7409c71931b3977cd4d7d18a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9070 zcmd5?O>Y~=8J;C6O4NrX%d#BhquC-+L$`^PI4Od-u8~Am5XZ6PF6GEo><~-tNLq^8 zrFVw57}$kdr$8?O`U6^^K@p&r7U(61qKCFXfSz(pe?YIj^;)2Po_Cg{9QOxM){bUo zXJ_Z*ndg1pcU}DV%;ZP^{48jy{HKV&Z$41U{27@@sU4)AiVCXjsl$TWDJYNSqG}h_ zPEqb5Us5}zett~tjP>(nwNvir$JNfb#>Lrd2ei z9(rnLMnz?%PpDnIa$J3))I+6yyfdpJv~#tULG7H1CRx}y$>G(iQ<#AHJCey-+C0$K zH#)ncGrtiU9r`k{7JsH5yN}b2_`0wq+D`mZh07&mhYoy|&`+yVY26bLZ6UDYRIViL4Fg@m|_!qFR3V z=C>c9|8vM_EHpL+X@Q%{bq5*>k^vNok^v4%k}0d8Rvp7*<`BuGkDsmZ0BhO|Gwr9l z!?k^ECH*jo{IK2jGkv?OjWzPL-4CsQ820=y%i=rQkGdK5#*eM>+wm@Tt`lMp_?~77 zZ{)@GS$lvXRFSbtwLEptBg*b6RWGP}_(i6u?%@|1OhN%gob-kaJ&={3L)qbJxJZ^W zYjRm#E{vi~mOcux>q(Vww@iSiT|3Rjvdpd@*)v!179I@7ai2qUiTMd~t7*rsw$o#Oa!vDy&~vSebbU+RR-gK$st zN#J4iI38Aqc*p^QXD2X(dOc2JTd$u%-Vj#GoA)YSlZ(QORkFH&LdLBwV1b2f71TqJ z26IIo-|P$Z@k4cVk1LY>M)l4}qcPPgOJ(yKv4G->Wppzdba1wF(a+y2wpZ-o>E&x(oXjT8If*u_W~}JC64w1 z?Y2IKBVuGL>uSM(z(GGC&IDL)%cM_vZy<;NG;R-jTWhBvx^8h5G)YS-N z*+LK=aEnI~bliY?{b`KOJc&ekr@U#e2;ddHGu~3+lvncRP&ZQ)AaYW?1c*OHruU3& zGd6R8MNefhLy#DNT-+@wTT!h^@*OHc zb^zwR5&?_5f^bw#3!V~B-N35Xs@(n!^BtrPAURRxI9lcYntf2=c>wAM1Qid{q(H}^ zfQ()rG!qB}1fu}U^o*QNebmJH+)V`$a;k-I#0IeOZkip2b}I{;x`D$Vkb^9V=5ClJ zAWmuBO5-FThNPEw!ZyIc)f8|@3w>0#{mdPQKGZaAf_>3pfL7+>-s3%@q^@D*%UZRPrP^YFrvk^a<)RAtjp+ z<`Q`X^2rJpo|I>%B%M}(N$UjB6rsHUjcD4w{W*Ha`zM6aK4ZVAB2SUH@H;C& zgMrO0D<>OTsDY04ZR*E~1>(9a?Dr+h3#M;H{dNkQLWxw{tOsi!Y+b&(zA5_{$1x$e zQ6)n;#u3v+>qeF^`&}EQcaz{G3OEnw;oa+7_2ANWApIF_8!%M|KvnQ8n{a_QYr;AK z_d+W6LxZtqPOoT7KErsVSlfKRUfW!|vc45O&*r2{W3z74L662uP&%r_#U&Xm{&^05 z5eX5v2+mprW6dIe4!kw)Rk=91eNAA{>vdUzdi^ksx@{KOpk9yCX1yM8y8?D5;0|6! zZg}P-Hv(!QAmeeL4VkN4Dtnc&%6w(oQaL!qQxhRx-xAqP>Dk0B`5}?K{eU1tMjw}$ z!n&WpC4eq-Dih+;sZ4knqB7w#umPNmvWQOj2o-S~P&uje6upK|;6k7X2SXeOI2aJ7 z;wfCh8*LUQrUQG~_Z7gKzO%K*w7q%pn;rnB;KQ|`=vrx#GKpE@K!}BuK2Stt4jL8g zDo(r{W=Ks~k?ea+lS*!HAb9SBYD5=d!uMphKm+&yq6ibt^H>)&n*tY<*eALIHHZrZ zlg64EsS!tNi990$EBjh3m{JE5%KU|SM<-aJgQ9YG2aTsu=J?@Y$>=EOZhx)FL>TtJ z=yoAMwk8C&VZM!RfR~h6`Ul56y?kGt$`zy5h?7VN zEz@xRM8wPSj1P6%wQ?tciQL!Fdog-driA2`bFZ(+J<;d;DLj<#qx!Cnv&g@+Q3I50 zw?p#|?lo!q4!j=F^Psg~We-REcUC%FS?!(|3by3o^M+&iaM|2CzZ@B2YuPZz6Sf@X zKs3J0P)Ai^2eIxP8zq5L5iD(_i4LeloF^lG2pnIl-Kh*Hu!L%br#SamDXGcdd#)l z8rTkmH$1(rwhL(ENa7k!n2z9#{zz`n4h>py2+omjrrpF=dG8+jtNs2Erl2sjsN)*}O>C2z}*+Q)7r9<;}tb&kmQ0XG{3{EHb(8iUsRC4qQBugL!65?UvMHO~Ni4hDzrG zjyU|kW!Q(|$&@^>YAy;eg()zKc4!PN`YU)WSZBiJ3oauOHGl^km`MR6>zrk}lOZmE zAqsIZBEaR99}ZkrlURZE=tD;)M3-cE$|N@mM~$$!&<+n9QFxJYDuYWkd4xC#@`}vp zAILmn&yT*4!y@*aN9#)oEFskKWk0;(%yL2PNx%u>eh%VZY&)YGzZE}JY&(u3to;&9 zCXfP~2#JG~K4(cS(Fa8tePC?AP}g{7;j2zb*aL?2*9!gK{+s%lVx|OUWAA^J-UZ1n zs_I?C-5a>J^I155M$R`+%7(-Whk>BJ7|H{1^cTsEU{}29EM1o z3hq{z?HOn1*w1{g z`RO7dK%e-rjDbM-!5+Z{MzAh>Pml@!w_ihPo12k-JRlJt4ymu>nPceU+Z~YM^g=qhkPg{pq<|G%$B4ir$}kY1#uz%8qrmWhl_LA_%eDpVB$mPEG=7Cx+n@hGj4oj0>(;4%Au3`S_BfH{_)#&#Bf)R$aQa zEOZN_Ki^kHUy*PJ9*B4QD}~=}Fmg5{-VGjKknYeuUe#9zWT+;BtwR@tUT+S^s>rAR zLwUhlcf-RGkk%#qeXVC9}(7|Gly6{ z(de3?oKye9wXT15sJOVQ_C`}_>0$DCVP&cVm`j^;!9(#fshQ@nT&yYDrS^MTZ z_&W^KP|he9F6Zo6l-Wr*&rQ&Y6u~c>* zUQYF*yU(FIc$LWoCL2sPnFLI3F(C(vqwC`06DWe@J36zypu>9^k|EqBHv*y{cn!n4 zf=j{#WuSR_u~M#-W=~g6Og}L@CfMfAQ#Hx%=MrE*;J`TyGRP%@@h4#y@oj{6p=2FA zdJD|Ca5y3hgak!C20uhu0@;ERi5BDo$2HAsiy%S9s!xC>s|+^DNy($ryItGO5S`{hadVdi#b?ev zhW$mAgFaNS!bH>)iNt-Sq?~a75#d;AVW@OnsRx)F1$;?tB&BR#^cZTGUyEDs~ zS&CF7qp?-TJ<~KrnxqYypy?kekOXK@6fF>>7uuplQ6w!86ewC0=|5@E073oxWe%7j}e z>`=2il$%jYJ<7?c<%|kHs@xrFsaMVJRDPG_(PpP|+tgB@a@wTL?ti__DZ@>(*2SiPzzY-L1~Pocd&72816x}bVwQ;R&N-}@KxHPQ(8RU z(&CX#ExM#dp{2!~qR_ zM9K%%wEsnwbKk9&yH$8QYZ%7*?G%NM2nzim<6ctEQSR<_gRmoV9#hrlRqe8xy{u|4 zsM!}(?L{^FqB7jeYH3I<4XfD^<-VwvMkOsI=|WbyBdUBw9N^DJ+{4&^+-GAGyL#lSZ$w-&^%IWf27R^n*DG)IE^dgv`I#^&6*ITqGj z-*jB_J%PVq<~=^X7}~dkaMknO0pE>J4Rm||jh|v3v_{4Ik&FbskHNgaAE^eB>lDH; z2;&~qOnEgoXvDf;k2I~-8opPt*;*IpP`qTX+Ho5)w>(WvFM;iSCG3>o(qYNVC-%aKcjB!hBd(1ytQb@X1wT{ zW0U5p8%6+@=|yHRFNF)ZWUfr}ECv>C8%wJ%^MH~}{yyAsXeA@J4 z6O}=A)s19?*1l=SI4~E0IkwM9lR`bn)S~NS3_W==j>A@!z>jUuk4)R_AMEEWSR@J~ z&v8T7F)`$v%Mrc$>cw=-)3PwzcQ_)tv<@=RFKfjp=y5^M4!|ij;1NwEJLt6&YKM_r z_m}&>aWT)c5){Oq#_B^FI2KEF)o$ccC`^P&y&y)ibzUwdNwkdInV=q@ zsRk9hdS=|43+-_I%#{$(zZHbb(HVWW2IkX}GdJ|1MQ6?rJaeY%&7G;s?c-DA#qLve zyRvLAxa=7Q7tZKg%S$_ehP*kl>|~S+c#c@8I(m#ABgapsLQLe9;pJdH`Y$9O#-BkV z1$2BRV{$BN#QXvRsh8ZI0C3Oc4I@NNglxNl545{Zv;f%VIC9s#k)U(9z5}r}b z+_e8Om5Xy~32YpM`TGilkUfR-N1tgKL5A)cQ^p zx2rojJ^IQQ^ynQj`ZqW_ie~Vc=A<3M`BQq2RpFX(RWsO5DNi{l`doUQEH`e1K+fbI zCFe>E*>sBr%&~cMJ!qJ&Z_ia-spYxaVbJg$jjuq|TVAzlHX;|;4%!-{;UZ2ky%#N4 z5a>J4o(cYQ{%R1^=h!+~Me}0mG2#%Un#~m+^@ zyS8KI<3&;iAFTuzWq-Uq=~Ql%(1H?fw^vVa>iLVDE))>mcRk(cw0YtFx(lbxvn;7% zfQgb>Sqwns1xj;n6k~3oK}G&YD?&)HdH^?j-dh-$vQab2MDn^MwABa3q8zLjz?ExXks)W#;od)LWqk%%&a<$X%6#0F2!4h3co^(Ug{F@)KJkE1;K%x zSjksZ3?UdDvB>lWX(TKPjS`yWVN)u40!bsGR;zZk;mR@D;X(rpw&hp`yh%1Eu{(GE znSqF~IXfV~lD{x;p1*l=NZcAU6@D-5ZstohPDJf1&$_T1owXD^=B$>6j2iMFn|a3M0$qmfV^P=KZoFgAIk z$ux_3LJ3H%6z`eHN^;2rsQRMCt(d1#DhD8}|6rboQ64kM%H)|X>x{N1{UB;8`dyTBI6S+=fkI|$l1k?z7 z)REwlYcVaMEJqT?pa{l?0;EG-pof`n*SW07lS4!4qlDOK%S9A@$)bDNGADV22H1uOD)jzFeBH zybBWu;%VftcoXHcEn=%h$y_eH+~Cox*3dPf49NuPWhVil7%2!*q5Q6|F0Z*)FDElv zk8(6ZmF+cB;NhWPOu^kMCl+}qbJEz4L})D9p7V542q6)aJ3M2oeNV+%6~5D0FA&7IFi>m&58ew5g?rVZ?jWyqB6O+q zvD7TZWO#;o1@lc#uUvx6ht7~lAyIywG9UOB7&Q8WeW%skQr(YQ12=jJ78V?CI~X+> zc#paxq9Ib)oyfx9z&1M7olXVqaJfT;9Yze>6CQW5DMn88Fz#u$TH2$WF5Y978enS3 zA_vf5_8EMp(QfT!gS)avRND!^z z#u39kSUF5Pp^AQ^HV0W5gmi8_h5$Vu1~pV_nMg~SW1^F~Yjx06YNVj3x|*u0mi7ZP zYSinHpN0Nz8Z#M1G*vJ1`=LyWXkKyaG+}U*(Dq>TF!R(PL`U$0*o5$amE3k=da1hp zLcB-?yitus`Gp2zC(n9Pj1tu#>0>ikgs=5~j%P>97#b?esR45eYp+A@^6aYV*s(2L zWxPc@f}#zZK`DpK&{mBTU7@*&@V?cDS0ees|FC(#u zyOwS)5F^n*lPn6i!QI1^$s(y#y@)b`p4Iag2F+L-e7q)ogoxn>!na#mB`k>W1H*|2 z&4uqSe)Fy0|Ng&S_HO<=4*QY&2PHgRKWHp0%D#*FL+J{l^+-gUdex4p`2!lAA{kpLQH5sp)l5FnG0AiPInC@NSm z9F?uYlr>f;S?HpNjgod8R#-(;b=|skF0F^5M}#-KUUz**hFl6zMj^Z9kn*J3)*1E^ z_Uqktg{wn#OUS*9Qi~S252^%}MQ9q4j8nKaIrU0;_?4+bNqQ4)#E7ycJICS3%B06O zGBIvBL0#hIhwvE9BT*e$qdocUgTU9`(FJb5AA(*E@^_cf2C=UXpIydbNalx)-NwNT zTl8g)qF2u7HTI!Ci}q~Y1rc#``$H%Z-blbTdE@(l0;K3-!lnEGL&=+J9i$tD84`FB za8!WQPJ`6K2oFXH-Zdt?3wor`e)6uQx?QSYmuj#m=#1c9TFE4<1DyxokWL9(LdO~Z zYh;384QqeJIl~*q`Y~C;j${eoo@6#T^(d6y>6MRwo;H{A4-YIQsn$!egeh!Z3fL%w z0WcK^v|ythnvFvKg+QR$=y_qy53ZMNv`;NT$^uu)CTw&!v*c9ZZXW>~{fL?-(6pWA z0Dvqsrv&r|QHCW`sG2r^+(*PqtmIvHYY_}4-~jb7fIS3H)IrUgHzSMz`k6m<^CpNV zDB=Bt9>mWN{HNw%z*`2LGHZe3YRZ0v)>D{LY`@~3Hm}(gS~zCR$mQU$X25>Ah#Ly# z5May@cJ{*1)uFMXDY_Gz1fZR<>+9jBXpEU}_+kw==dmPEX4-wkFv6)x_!k;gJH#B= zHw{BHA^EQBxQ-sA7*XM2^#e3jNMp0+K|0Dy!yoGc5+T=3(wZl!7SaLUl+wDBEF)!0 z0H{ghyHT(~1}!QY)+HwRDCPB%XUG-T4dV2arI;%w$15Ht5B%Y&+-&D2u3?_tTsT$KZ*PjE&$E9RsBX zsR)nAwS~GVS_m#eQ%0-9qSa0Z97s!WiBdHTtso*G98gXm*?}%&1atvON1Lc3B|>;~ zNf_|pKuI8B`d_60+7o_)^T|IDtY4IRREjjd)GZA^Zfp#^L+IM8>`GD;{T-#q@XxmB zwla{Rgst0QO^I5t$6$wcO5?Pq>FR?ZA1$IamMAknFoZVd6%B5pOT=W|pTmIeU`6w zy@|bBQW047SzC4xNm!~sMZU`G7Ccfk;#w%o{o@ZOPL^lgJM zhaLs*mZCPQBvflQT8S1%E%&1t;278hyMk!%R?A%~{6Ypzm-Zxjh71d2J)Yf+8sExn z90mvbR)&fZ)}?_21;_AvS+KN^W!RXp4w8@O{mOxR8_&}#$FsTkfN;M@=rhIh3>O1- z)}BjRLw(U4b$X>emm>rYNrz01n#p0c1jjw51VHcZP%$nQ+@#=6Sw$TWcnMG*701R8 zv&klbmcNi~?mRxG8~{;Ub#iKLPQ}OJxyZ)w*<&f)R78^$ZF1`U7`9j4-Kp*hobQ11 z!E5r3?BvR)vuJmxPo6)W)hmTlT^}iG@i)Da&AkqsZO_R*p|Ty$#U{rmJh=64a6p>s zqN$Vvj)bla8Qe~p9z22bHk5@_Re|pbvNH6Q&^4DKMA}sd`Orh?goB}9+f; zP5tx&_9qMit~vVXAd!Nhq#iaqieP+pP=4L_)=Vd;!IhWJi1wko(tvqCWl9s^sNpSW zI#7VXOyOm#nsKFm1_rGSg$lO+gokNo!|KK7DS4CoRgyi@TL))g(kHYzEHxP^Ph5V@ z&aFNLa`XqRn>3ff#{gammgX;L7(|1BE`cB*N?>4CY@dq6TmXRqTs?wD=*@csfOfS; z^rez7YTG@AP-16`$PER~i7kx4rV&NBF`2RMF%lTI(*tIoq_hdv`(f5dTeUrIcOt-L(|w5aqZ1! zwpVqXV*Ui1Q3bOoJy=gLc?XkzB*NI^^|~u!f;K)=z7QY8*o;^u#Xc#DN8G4aU2V8L z!p_unaFoShlx8(?gyPY10dD#g(t=_z<^yfugyWXWSH~xYhsMjJg)2kX$EV6uW7i53 z*Qe6K5>ZDiW!(9Mf8FQtO(I4?R?9+BoS@D^kQ0|lA=QDccVY(723y1+k$EWYh!$0| z*Q{sxKww2~6_$on0!$bDa(Zd=loPBn6AmnMh1?!-OP-qd+algckq@;{Zx?a}jIyiR z#p6SWC6u~Q*hB|0)M|B+Ccm{+IZ*MpM4|q7OaF7XQR<@cRuhDh_^-Z==ji8=sLme9 zSSRt@14-)$G`DW!Nag^vxg#h!V02}UW%{-*g+{m^?RuLZ>U=%W@7j&WjD4A2NN>AQ z+kt21cD3yiy^yV_TXIHv)(>a8WrPZ0P0&)h8^I@<0zje9)K4%4=ced(qQ%E8c4n$% zXbZDwPKD3$I>7ZvF$9@*8dm*(A+y))5(WmUmb*m+gUvF!Bi-s-bXz$|#fzMmwoQ>T z$dbP*9lp$)1EDObJ_u>7MFdM{eqab~82>>s8`hTe=Req0F%$k05e(B~3RyVfn+|A> z=kJw?H$rtzIII@rmX?RrVI#tViXW>L-aW$ zq7FBOfre8z=fc2tDmJ|Anqg`N*W{1ez+5+TN0PBES=T0hsAZT|ZrIAMTDak$`9x@) z$EY3&n0U^mw$=a?z_=+BoHoxtCA@=D&IO$<=IQa;iYIBMR%*pFc7yF^n25_vSlN?! zOgRQYjpfQC!^G%7D@+M3FHMw(t%;%0ks-Lm5oW8bSDCz@$p@HF^bqmgLkh$%&Yug}=~rdCd-Cp#|m8!4{g;n)6T%Be5GN;&0&d+%lMc}!=Aj`$$>c)kD}G#sq^H!C^3}rBmC+<$l6*JIN6RDQ6NQ<= zh|sVNp?%)bBUJ5$Xd6k)`>@7et3OK6 zhf4p_P;qp;aKB4dN0x9n#Cbc_syNuDy(AEvlnmum9-}5DBU2o~M*snf8IK(t0TL=? zU>tQ;MJH)KMTs`9!YowEvsd)fpR8~Ov7)y`yO$w-y#(7!)g4339QAkDW~|$o(Rv-p z+k!jr=zg0=M?k2|p$*P}LJd#WY}^8}_Mh$+n0cpBtF50ZYK-MP+sz{hh2JF{ze_Uc-YuCP<@PEUspv;|H_A%X0Ip}qycNCN{ir z+Q&cUR0sfm2&@r&B<2x@S=>0-E1**`BS0w$j_VkE8OVAQ?e*m*TEPY~-%^f}Z3?94 z{>t#)dvUq^<>qyOvfm^x&FethO_QY|swfVqL2>9;PR^gJR; z4@=t8SI?eKwV$Megrc?{E@cjZfw0$Tu@O6XNk*GuY0xUYbbYF9jZRyFO%tVp1O=aB zueS@H%R>H`K%=0fV6Zy~-7&Yty=8D%6LN2CTwB2gBDZknx5M3)h5Tg;B5;Z6r+I0c zdr4ekn=h?wi8hCpOH0$j_91v`HJQ?(Uw_@QL!x>5ksbc%(|RxW!amkEQ(7V(0wt1N z{q4rCiupcN$=yQWEu2Q>vVfpmMpQ|o%17QnQ0bt8Ja1gN994rjDwnl&>MaymKgWdh z&LV*jgx5mkFY)o0k;oKUnvU1SezbAOl6%Pv(K&1ekF>dExvy0Y@yA!$Z} zsVD?w@sTs9XbF-HWevEkSrNi&p^IrVEkl7p-b8{oUPFbBgV46=VKKEMDwi*yC8nO= z#CV7Vl~=pqHEf78$E4wmZbTSL7lQ`WGrIU1UMxBr#B_1iQC!Mby=Aw$e%eIXBa}Nt zYBuc9Mhw2|B=1Q{3{M5l6*~GVL7hHAi7cgt7{v}==YwGb@uu+rlQN@6IlDgo0nw2} zbI>LwvSMu;Bz4tZq>IRjlA`14JMl=Rld6xLIs%>QEU*B$?Y!DNXTG{HiEt}Ga(%;d z_I08Gg(j6{bX_+{bf3{lAB9r#nj}q{f(%3iUI;;k8|X%4fEe7|Lke@e&D&WKH8U>J@hZ=o4H~@9Fb=H6 z*h%NR4Fq-(HAggM*Z_h@aet(dl`(gb$T13PczU#)9+|GG;P!W3$4xWpS;D5tU?ibdP8fN4jvO!ZE z*1`w37)09ul96QCjYPsPAXGv7iNx$qGOkVXa$AY^+EaalIW9wJI^wJuF>Q&Jn5H!A z6HLUND+0$&KH5x9BiS(PSP{#%_c99&`e$C-cVICpdkJB(kBSR)7aXj-#Je&_VZmp> ztag;{GQ_B*UD}CzqFNQ*iMyqhYXqO@JxJE^RS}3)h4c#C#1+!MuoJuiDD<_E)(oAU z_V=~Msuog_7I!mwDgGG_4As@Nb!5Db4drXo2&|V7QK817&7UTUBerE>0V*?s0%a=LO z=8s0{H#z9A(oqyz>|~LZSY()bv|)a~17$TGh0jN;i?wXh{K5w^D%~J0ve?JE#e@^; zKzQs> zeneXRt$;~`iFg@bVGg>0vOdVQ~Jh#*^7f&1w6gf%Mt5tbxl3w;=#m>3sIiI-hi zDiWi99g#%T`ZyE%NkvWoIW0-JOiElQgKE|KEmUt@Ey+Ff`$fQ9((>G8B5m{#hLke#rj5i=K49AN%{p~6(C-5 zW=X1UL1J=tVtOh_XHmY1eo1O^iGF;1W?p7Ve7qh|EeFs_o80`A(wtN~ki&|Bm;nG@ CkTaJ6 diff --git a/contrib/site-packages/Pyro4/test/echoserver.py b/contrib/site-packages/Pyro4/test/echoserver.py deleted file mode 100644 index 9ea2a631..00000000 --- a/contrib/site-packages/Pyro4/test/echoserver.py +++ /dev/null @@ -1,126 +0,0 @@ -""" -Echo server for test purposes. -This is usually invoked by starting this module as a script: - - :command:`python -m Pyro4.test.echoserver` - -It is also possible to use the :class:`EchoServer` in user code -but that is not terribly useful. - -Pyro - Python Remote Objects. Copyright by Irmen de Jong (irmen@razorvine.net). -""" - -import sys, os, time -from Pyro4 import threadutil -from Pyro4 import naming -import Pyro4 - -__all__=["EchoServer"] - - -class EchoServer(object): - """ - The echo server object that is provided as a Pyro object by this module. - If its :attr:`verbose` attribute is set to ``True``, it will print messages as it receives calls. - """ - verbose=False - must_shutdown=False - - def echo(self, args): - """return the args""" - if self.verbose: - print("%s - echo: %s" % (time.asctime(), args)) - return args - - def error(self): - """generates a simple exception (division by zero)""" - if self.verbose: - print("%s - error: generating exception" % time.asctime()) - return 1//0 # division by zero error - - def shutdown(self): - """called to signal the echo server to shut down""" - if self.verbose: - print("%s - shutting down" % time.asctime()) - self.must_shutdown=True - - -class NameServer(threadutil.Thread): - def __init__(self, hostname): - super(NameServer,self).__init__() - self.setDaemon(1) - self.hostname=hostname - self.started=threadutil.Event() - - def run(self): - self.uri, self.ns_daemon, self.bc_server = naming.startNS(self.hostname) - self.started.set() - self.ns_daemon.requestLoop() - - -def startNameServer(host): - ns=NameServer(host) - ns.start() - ns.started.wait() - return ns - - -def main(args, returnWithoutLooping=False): - from optparse import OptionParser - parser=OptionParser() - parser.add_option("-H","--host", default="localhost", help="hostname to bind server on (default=localhost)") - parser.add_option("-p","--port", type="int", default=0, help="port to bind server on") - parser.add_option("-u","--unixsocket", help="Unix domain socket name to bind server on") - parser.add_option("-n","--naming", action="store_true", default=False, help="register with nameserver") - parser.add_option("-N","--nameserver", action="store_true", default=False, help="also start a nameserver") - parser.add_option("-v","--verbose", action="store_true", default=False, help="verbose output") - parser.add_option("-q","--quiet", action="store_true", default=False, help="don't output anything") - parser.add_option("-k","--key", help="the HMAC key to use") - options,args = parser.parse_args(args) - - if options.verbose: - options.quiet=False - if not options.quiet: - print("Starting Pyro's built-in test echo server.") - if os.name!="java": - Pyro4.config.SERVERTYPE="multiplex" - - hmac = (options.key or "").encode("utf-8") - Pyro4.config.HMAC_KEY=hmac or Pyro4.config.HMAC_KEY - if not options.quiet and Pyro4.config.HMAC_KEY: - print("HMAC_KEY set to: %s" % Pyro4.config.HMAC_KEY) - - nameserver=None - if options.nameserver: - options.naming=True - nameserver=startNameServer(options.host) - - d=Pyro4.Daemon(host=options.host, port=options.port, unixsocket=options.unixsocket) - echo=EchoServer() - echo.verbose=options.verbose - objectName="test.echoserver" - uri=d.register(echo, objectName) - if options.naming: - host,port=None,None - if nameserver is not None: - host,port=nameserver.uri.host, nameserver.uri.port - ns=naming.locateNS(host,port) - ns.register(objectName, uri) - if options.verbose: - print("using name server at %s" % ns._pyroUri) - else: - if options.verbose: - print("not using a name server.") - if not options.quiet: - print("object name: %s" % objectName) - print("echo uri: %s" % uri) - print("echoserver running.") - - if returnWithoutLooping: - return d,echo,uri # for unit testing - else: - d.requestLoop(loopCondition=lambda:not echo.must_shutdown) - d.close() - -if __name__=="__main__": - main(sys.argv[1:]) diff --git a/contrib/site-packages/Pyro4/test/echoserver.pyo b/contrib/site-packages/Pyro4/test/echoserver.pyo deleted file mode 100644 index 213edeba6c7dda385ea7df034402d4b7cb0f39eb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6168 zcmd5=TXP)66+Sa7t+ng*1=$GOTs-&^uL-*X0!2^(As7>!kcAo0TL8(vzuND!(HCArE=TTZ+mLNmVNEdCWudeWzz7#ZK}nWKVBjPIsUF&ZW=k+CR=T zfBDzn#{;SUYWRH*pZyDpRAdchAZaM0K-N(4^-$KrS-mD}HLcfVG$-SES(}%DbaUFS z%Ua#HvBsRN%}F{hTUhKAD`r79L0Xq@L>`HJvDT3El$d!LHg#a(aNxA04K~(VV$Mj~ zly9)cSuy7%JuN$-lz$e3W+21!lD6dPqd zoUtyNOv_1OjqNt?Z)Y~b|EZlOqtRZJ<-5g>Nu!Ou$hxF-S-urH(u|9AIxs@ZtHQ3#6zTRx!bKEHz*&@R3Hrv2@ zuE5s#*)~{elvum6PL5ZV;dQXX;W8Q&scCLZ9S9R$B`;7nWr;=hNU}K{b(>A@9Cg5e zvJ#td;Y{?&#?TC$?MBg^VzO6eTicGuyH}1)9;GJwX@N7gGdABVldp?%H_J^oH|}<~ znf(iYy*7Xp*JQC>nk1dNY~=gu=*yEaP}1Qe)(_Q-Q{TsD-#}p?O$Z-lC3>84yrbwopdzd@p%!|)LtrLlv74okph`*ssVH#fW0LIZ zV)r(q%-LuqajslhhmLGOTncY3PzR(A(-b!Tf2kL-bAr4G88FaIHW!72eKK;(oa5u zCZoRE<}`Z_MQPl$%y9%LVYy{3Rs5!fvf6}er0 zt^4NkDBD=}#=w@%oas!G!45IPOqj+pO=FpCd(-HGnyj+tdHh;xm4uDpQZS(U@FNV0 z+;vncFVo9X8v_laXrYapUC3Hr!TPJk0GPW+<}kQqRinUYlG# z2BGQ$?afim$5ZS2oJ9KX4!R3qZN}AY)@ORV7_=K`I}-H!@QfICt{;9ZSZM)mK@dR>H+?Rje4W5XY1jZ{@NN~ zSpR4T@8h#qP?(wT|3=+^i?AOmWth9A7|;Uc|W0>fcl z9VlSM^&yq(?5U}?iKfDArxR1gPqFExljWI%2#7Qs{Rau|%v^`yog76MW5mzByK8c% zg458zq{_G&dQ4o7gmg0Uxd0ws#b>kcC?x`1hYB5Uoj|_y<~=iV zLOQSRpf1BxL{(9rli`9k6kQ6pr+OZY*jj8-u?0{P8oK zNkU-nsQ0CLFvacgaZyaZJBCgM%W3{5`BWPZZU9@2OJe$`_$&*;AHRU&NDs7jGRi;| ze+8eV-0MN3u2(57jo%%P+(H3Gpmhau0ChkR5_oBAI07?!MA0)0@zUm6-gxM;)C#-^ zIq)#a+);Ev^KslNKk?Xt(UbCeVNlXgs2*aQnLtfKlrh?Q2X-|p_1{7yOQJYa9kwNBy-lxzT z%z76ky&&mDNuTCC7zZhqj?a5W(o0WJTs}^5MN$MukFD|Caf;_9ec=g;tH&uKNfDZ2 z?`!Dw+|52=XK7Onw~BfTLzH7U~| zy(#bsNcZ}uXXGuJOoH@#G6YJuR2$)b=4JA`z*7#-&ZeBLmT#4RIvPVgZC4Z^ts+5@ zgSt|mm-26cdrCQ66n8;8FX}eRF!uo1x2yj2C9Nz;`hBfD11qQk_ zAm|ol_^j+VW&gA;*OL7fH^A!q4fX^T@ZW}MN5`Jyp&xu3=6XuX1Vdd9el!gNRWRawxENtt|(34A-=Vj&@9bX zW|7S!t#sSEtiK*TL%oV2Zddj5+SGRhb!$=b#GGBrd$!Z5GC|7=s;WoD)J>+&ul>MM z7@f|8X@+w$h)Ik5maAx@B*#kv`1iy+mR`Tp*)e;TmmO2lkAD8+I}tjF1D+x5OXy#n zJ)AJxyJe$|X*O~l;HGac$I^qYifEYZCaMQxC`5+*>?>SKz=)eo-Dc;9x*0d>|LpD; zRXWauH+~Tmx`c|Gz#bDCz5ZiwFqXJYEmJv_oktVdJ{D6!@Kwc+dq^vl>uLgCJtJ@? zcUP(CVG!hK2QgtswJair99wlYg&85Bxl`n6MjaS{Q2%(Va+WF!Y;P#in%%V=iy0EN z@l`qH!mN2BF@=W3Z;z7kMw-0)HCVCA84bMGYhT5WQsC)eLer0Lu!mAf(zK6WoIE~1 z>r;|GpB^;n(coOQP=f}TK79eP))9R-P%9b~`DW%{n^y0}pWlt|fAQ&EwQy*mQka~t zWy;D-X>_q(fj2F(b7V*{n;O2x*I84AqOJNoAm;l)%qU88A6;^0tMO%YYY%;pGrd)9 z^e52y&&n+3Lx!?G7(pTQK{BSTxr-?LTh1?F?J!2hy3i937dKaEN)4zzY95U3aWaVC z;}TSjRFC3yvIm%yA3cEQ`Wk2+)0~r08p@Gg^8@_abrceuM_%$=a4lHE=l?AQPvgC( zh5m)$TyQSDgtwoiU@=7CihqmYEtD#bJsZ(UNqt^fc4 diff --git a/contrib/site-packages/Pyro4/threadutil.py b/contrib/site-packages/Pyro4/threadutil.py deleted file mode 100644 index 524b3524..00000000 --- a/contrib/site-packages/Pyro4/threadutil.py +++ /dev/null @@ -1,19 +0,0 @@ -""" -Threading abstraction which allows for :mod:`threading2` use with a -transparent fallback to :mod:`threading` when it is not available. -Pyro doesn't use :mod:`threading` directly: it imports all -thread related items via this module instead. Code using Pyro can do -the same (but it is not required). - -Pyro - Python Remote Objects. Copyright by Irmen de Jong (irmen@razorvine.net). -""" - -from Pyro4 import config - -if config.THREADING2: - try: - from threading2 import * - except ImportError: - from threading import * -else: - from threading import * diff --git a/contrib/site-packages/Pyro4/threadutil.pyo b/contrib/site-packages/Pyro4/threadutil.pyo deleted file mode 100644 index 71e6c5fd44b9fe929fcb24c14bef4bdab2eef480..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 801 zcmZ8f|BBN<5T2yH(~DfuH<Eh2 zNALl3HnARDAYU>&KfZ5fw)guu`u6MRG>3oB0ew&DwOUiA+kpv$)g-N0lx{3%3s*E^uQx?u4VLh|-j;nc>Rel9Do(uhHqd zNee2=6$*#KqS6l8mI=u+$&=_~V>A|;TlLtrrSA|fun>mlPBx>qKviqwtXCLyqR?>3 z952YrD~nsf(3QkPx;K)eP}WhO3BJ+=Cj#Ga%b7Dpyi|mvWfjLb+mOEh5)HrE5OMJ& zi8_rZl;%nr(Ue!(ar}^7lPsGcRjwN&R;9zN!Lg~RRie7k#1so}o*MRDo2^hhQQT2Q z@rRz6wgB#sAlGUsR-Nq+pHAX?-sfRHFLXYiJHI%u``MI!z(oYJchmE;*W=0CVQb#* z$UPwMxSiCwF*L6upR!+JOa|P7U6YyW+NDzGOs4NeW|(QxH-@IN(`IecuJL5KKS@7x zW9hcobeKF(rO48{9gVjvt>L7md%#xQXOiNyquw|n6AHXfd@s7~y!68CfZ#BE7#;<^ KU=SP!kHWu>I_XRR diff --git a/contrib/site-packages/Pyro4/tpjobqueue.py b/contrib/site-packages/Pyro4/tpjobqueue.py deleted file mode 100644 index 02de2be3..00000000 --- a/contrib/site-packages/Pyro4/tpjobqueue.py +++ /dev/null @@ -1,203 +0,0 @@ -""" -Thread pooled job queue that can grow and shrink its pool of worker threads. - -Pyro - Python Remote Objects. Copyright by Irmen de Jong (irmen@razorvine.net). -""" - -from __future__ import with_statement -import logging -import weakref -import time -import Pyro4.threadutil -try: - import queue -except ImportError: - import Queue as queue - - -log=logging.getLogger("Pyro4.tpjobqueue") - - -class NoJobAvailableError(queue.Empty): - pass - -class JobQueueError(Exception): - pass - - -class Worker(Pyro4.threadutil.Thread): - """ - Worker thread that picks jobs from the job queue and executes them. - If it encounters None as a job, it will stop running, regardless of the pool size. - If it encounters a lack of jobs for a short period, it will - attempt to stop running as well in an effort to shrink the thread pool. - """ - def __init__(self, pool): - super(Worker, self).__init__() - self.daemon = True - self.pool = weakref.ref(pool) - self.name = "Pyro-Worker-%d " % id(self) - self.job = None # the active job - - def run(self): - while True: - pool = self.pool() - if not pool: - break # pool's gone, better exit - try: - self.job = pool.getJob() - except NoJobAvailableError: - # attempt to halt the worker, if the pool size permits this - if pool.attemptHalt(self): - break - else: - continue - if self.job is None: - # halt the worker, regardless of the pool size - pool.halted(self) - break - else: - pool.setBusy(self) - try: - self.job() - pool.setIdle(self) - except: - pool.halted(self, True) - raise - - - -class ThreadPooledJobQueue(object): - """ - A job queue that is serviced by a pool of worker threads that grows or - shrings as demanded by the work load, between limits set by the - THREADPOOL_MINTHREADS and THREADPOOL_MAXTHREADS config items. - """ - def __init__(self): - self.lock = Pyro4.threadutil.Lock() - self.idle = set() - self.busy = set() - self.jobs = queue.Queue() - self.closed = False - for _ in range(Pyro4.config.THREADPOOL_MINTHREADS): - self.__spawnIdle() - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - - def close(self): - """Close down the thread pool, signaling to all remaining worker threads to shut down.""" - count = self.workercountSafe - for _ in range(count): - self.jobs.put(None) # None as a job means: terminate the worker - log.debug("closing down, %d halt-jobs issued", count) - self.closed = True - - def drain(self): - """Wait till the job queue has been emptied.""" - if not self.closed: - raise JobQueueError("can't drain a job queue that hasn't been closed yet") - while not self.jobs.empty() and self.workercountSafe: - # note: this loop may never end if a worker's job remains busy or blocked, - # we simply assume all workers eventually terminate their jobs... - time.sleep(0.1) - time.sleep(0.05) - if self.workercountSafe > 0: - raise JobQueueError("there are still active workers") - - @property - def workercount(self): - return len(self.idle) + len(self.busy) - - @property - def workercountSafe(self): - with self.lock: - return len(self.idle) + len(self.busy) - - @property - def jobcount(self): - return self.jobs.qsize() - - def __repr__(self): - return "<%s.%s at 0x%x, %d idle, %d busy, %d jobs>" % \ - (self.__class__.__module__, self.__class__.__name__, id(self), len(self.idle), len(self.busy), self.jobcount) - - def process(self, job): - """ - Add the job to the general job queue. Job is any callable object. - If there's no idle worker available to service it, a new one is spawned - as long as the pool size permits it. - """ - with self.lock: - if self.closed: - raise JobQueueError("job queue is closed") - self.jobs.put(job) - if self.jobcount > 0: - if not self.idle: - self.__spawnIdle() - spawnamount = self.jobcount - while spawnamount > 1: - self.__spawnIdle() - spawnamount -= 1 - - def setIdle(self, worker): - with self.lock: - self.busy.remove(worker) - self.idle.add(worker) - - def setBusy(self, worker): - with self.lock: - self.idle.remove(worker) - self.busy.add(worker) - - def halted(self, worker, crashed=False): - """Called by a worker when it halts (exits). This removes the worker from the bookkeeping.""" - with self.lock: - self.__halted(worker) - - def __halted(self, worker): - # Lock-free version that is used internally - if worker in self.idle: - self.idle.remove(worker) - if worker in self.busy: - self.busy.remove(worker) - log.debug("worker halted: %s", worker.name) - - def attemptHalt(self, worker): - """ - Called by a worker to signal it intends to halt. - Returns true or false depending on whether the worker was actually allowed to halt. - """ - with self.lock: - if self.workercount > Pyro4.config.THREADPOOL_MINTHREADS: - self.__halted(worker) - return True - return False - - def getJob(self): - """ - Called by a worker to obtain a new job from the queue. - If there's no job available in the timeout period given by the - THREADPOOL_IDLETIMEOUT config item, NoJobAvailableError is raised. - """ - if self.closed: - return None - try: - return self.jobs.get(timeout=Pyro4.config.THREADPOOL_IDLETIMEOUT) - except queue.Empty: - raise NoJobAvailableError("queue is empty") - - def __spawnIdle(self): - """ - Spawn a new idle worker if there is still room in the pool. - (must only be called with self.lock acquired) - """ - if self.workercount >= Pyro4.config.THREADPOOL_MAXTHREADS: - return - worker = Worker(self) - self.idle.add(worker) - log.debug("spawned new idle worker: %s", worker.name) - worker.start() diff --git a/contrib/site-packages/Pyro4/tpjobqueue.pyo b/contrib/site-packages/Pyro4/tpjobqueue.pyo deleted file mode 100644 index 4df08178edcd3c461e242048d5106afd8fcd3250..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10078 zcmd5?%X1t@8SmMbv}?)oJ27!c=OMff_R1uw3WSj0S1JyU?2MeqD8Sb2PHQ#NXl6Y< zYe^}m$`uC=TvA0*ML|&%Ip)rVDlSl5xN+jZl^a+70QmjBo_#2`i{gMK+0*-VPft&G zf3M&7^_2fL+xXExKKHw-_^aagyZ4l`{{#^#wSv-9VM)cF+AOJ+lJZzDtGKLI%5o2M zMXgkZbXBcXhjdM?OsSni!|JqJnO4im)2d?XYNMgDf%3fJcn#i6Jax;I)%pPyR@8k@ zt<0#fs?4-npH-`PZ%uuw)P1GCzj9E8Q_9Rq>-4AI^RxUrL^`kcP8ldq+T_RVIR8-2aAZo0W`X?-c}?_^PL zE!UkLeKp%Oi4Ki^Crx^~8L|4^EchtR?nH@cB_==9YDE9WU(>@(^Mfe1qI|7w^B_0q zoG;o#XyZFyY2|&)U8bHNMdM=nPTIL}Cy3&p6PqhpmS!WSbF_>1umRH`44CdIf2j#t zHA^URHn-bJuxZ-uydjm%G#tdJx?X5fLfH*a3u!-Jh|_KmFIOvfK7W%STtl20x=lVf+Bj_1EiFGZE*VXFp_?-H2T$U59 zc{%xrg}z6Y{ZEVZIe=eAvDZtgU=B~>8ZM~@dRgS^L56AEa}1-dmZ)e@5GrO#hzc3W zMcMP%sfNa%8>$ddC)NF^yJ0EnRuiuDQPMlBGt&#QFgDgwUHBp@nzqqL<`KgNI)(tS zx0_&^;SF{z&2rs0S(JvOSIM-39IDXIb)N2eGbgrXFr6rY?rO8T$^qEPDKTf_v{okM z=F+81@Zt_$x`$m{1d-#vr|!ZImDGBP*2v4tthXyFud4N$y6e$)v1N)Jf&Q?!r%te` zPP0k^Hcr9vaQsC(hm!Jr&avm*e5jiIY09blzCsX`T_B+8+R6|~&U1}3M7gaM38 zl23u$GQmb>RxvB3_{)CW=A7?}La9@mg&^CcC$`tAJG)yGulk(8SQ{R@ZgVNmb1pPF z$~)@Sy(1{Q)OoIi8zNl!3yAGMkulkyxwY%qf2BSK3p{0?Q8yB$c8;>**u~8+!w;QS z73={0(1-Lxyk9Lim6g?v+dojZ%i@tZBveYNjcJwLm4T*;j@}LQ;gdridpl3b6v`qP zyi*@`T}Id9J@Po_3lZ`@QGCxWT+4XHlGygeCfDpwvk);LhkGV}C++w&@nZM#1E30d ze>;eCpF$+Eu!fomMHZ~dFAnU^kiH7t6E6FQIn3U2*a_$$c@a~wB4}Xoj92q2UKxy@ zEzNnA(v&w>n#J!d+8W+rl!v?~SwFVevbc6TOuOy2PpR>rKrxnFp&q;Va*n5PS@Nz{ zt9kXQ{1`|FFm4FG06)c5bpe?uW=`ND)l5hN8^Z4H1;7a$ARCwK&ef>8$SBIJU@ z$Ydz7*gg0e3#_yW@8m|H1t-GLaT>rUbWFZw0H@+;lK>5Bz}KLYjB)dA|H_5SH?Cj5 z)_(WuqN^)0H&jGdlg34!PoLRMgkQ}@G!$V@rF!Y=lmD}&{vJ|uj;wA(i2|jb4 zY>-DWUa8czw7Ve$K-q>U9kxRnBA5dqI4}u-sD5FDnTP=yW7rp`bLZ}D|g_0C=GI>vxh66#w4)vZDOTcpoa_R za>~Ueta}#~w{TzvDxe?-QN0uDDXI@~ObEv?a&@ zhqp^8E>TZ(m~JHkb|)a_S%BVN62yST#GwJOX$Er!Xom(GvT+>gaOxnJaa#6Sj7v_i zBS$`~ffniV&rwlzWbMF&i%l{`b z9Q>F^v#iIloEk4f`SjX*$6fy{UjHF3hYm}10ET0L24a_o92V}vCKuebPm;CZ8JPHq zrvMALUmP>{zEU&UWvN#bh=RjF_+UdlvDwdiwANCB2 z{n61?Z3uiU+sX%qs6o=XQ@h!F_WqDvT&w=+`?TPI9wNjnlpP8oZCz}>O*$NL(O6qSlqjvB$=QlDiXkfK8r#`(O0Cw^iDABh#{2bo+nNWC z&h`22`EBtc#DhW*4+=q-@R!0Ou7vAK!zGSEha=90b;O{ID((b05xTo zi#@*&Bn3aihpL1U+o#oCj8~T4Kbh#{TEB@dx4Xzqz-hp@KrbuMOEJ5D#q1>VVDxbz z3`f2M;DNDE&m<-b;<3YMA&=Yv8bDaHgT!?#iC>*c?%BmK90_SW@err2PEt+&55Y$; zOco3LaIqR9ptCyAiP_T3_A!VfZa3j*W^idS)7SXrtVY>AChH^;i}LZy?NKah><=-V z^J(I}B5L%ah@LOzXdd;klk#6;m2&1x6dmmtr%0AW5+pLgV3S$Oy+bPkj{8~KMdI@w zCTdThP_+hPi8+84#uT$c!=TYl@fykHE2tE5IX{$3Sn>(^o0P`~>^Y^!ZJ-7v&>TQB zy<=qG#M1>~I8F*l59ej}Zo__V zk39@!JpBt?lUqT<#cL!a&!SQY$&sOu6cVxw^94)d!IE$XVVhv^U=N0wmm=8MS_2XV zP-P@xb(3+EJ=4-R*Pxb8V4O-9-A5+{oiyFp0478@(pq%8Ms>|uJR;b4`N;`wS!h9oSp9-5WJJ2}jc@JFh^S?|Q&7pofbq5JXP) zfbKDdkS4eXhLIt07+<((nV>E9+U?hCdfqn4aA)ww;`%b%XeIoEDE5uu7KasX@}Fa7 zMW{B1P>t7v`uqku6+*l)6k@nv=WiKkotN9l(z{)Tsak^A0K@K<@ZVsZkwTt8wa}38G27~bwf3R=R=G& zfPx{d8OP}s^z4DLC;i&DF@pNUimFhG<*Aga)#VuI>0wTX zMsZ|;KS0Gv7lK(}Ay|lws>)8%m@c`<2a>}RKmp#PGg+qTkamY)!GhpqjwY8>17av& zwiKy5)W#yVM2M>$>YW70Q30I=21{-wOL2N$hz*hNn5)grfz4qtU=un<&^W9DKi4?g z5Qj1&yCoaP6qk!5-XwK_g$zmDCFa*BCU7R1_D=-Wmv{rUy+np5@%p!LSvrYY6-SKq zdKv$w>oaqex$0b{Ua23cSL@Z8D%uXzXWd?Lhl=lbRtNBL_>RX%lS5!g0r%kD{018Q zSJ}z~rsA6-`L2oQO3u=VC7}pj-R$Fg3i;s1e~!=8QNSQw!(jMyK?Lo4XmaP)ym@82 zYmoFz6Bp4@&L!}0r$g>z2Vxx;ulii+7!(TixN*3M&g<^Or#I;5>?str(jhq(;?WR~ Srf0oF<+<|7@`=i+7ykpHGHaRu diff --git a/contrib/site-packages/Pyro4/util.py b/contrib/site-packages/Pyro4/util.py deleted file mode 100644 index 9c365d1f..00000000 --- a/contrib/site-packages/Pyro4/util.py +++ /dev/null @@ -1,546 +0,0 @@ -""" -Miscellaneous utilities. - -Pyro - Python Remote Objects. Copyright by Irmen de Jong (irmen@razorvine.net). -""" - -import sys, zlib, logging -import traceback, linecache -try: - import copyreg -except ImportError: - import copy_reg as copyreg -import Pyro4 -import Pyro4.errors -import Pyro4.message - -log=logging.getLogger("Pyro4.util") - - -def getPyroTraceback(ex_type=None, ex_value=None, ex_tb=None): - """Returns a list of strings that form the traceback information of a - Pyro exception. Any remote Pyro exception information is included. - Traceback information is automatically obtained via ``sys.exc_info()`` if - you do not supply the objects yourself.""" - def formatRemoteTraceback(remote_tb_lines): - result=[" +--- This exception occured remotely (Pyro) - Remote traceback:"] - for line in remote_tb_lines: - if line.endswith("\n"): - line = line[:-1] - lines = line.split("\n") - for line in lines: - result.append("\n | ") - result.append(line) - result.append("\n +--- End of remote traceback\n") - return result - try: - if ex_type is not None and ex_value is None and ex_tb is None: - # possible old (3.x) call syntax where caller is only providing exception object - if type(ex_type) is not type: - raise TypeError("invalid argument: ex_type should be an exception type, or just supply no arguments at all") - if ex_type is None and ex_tb is None: - ex_type, ex_value, ex_tb=sys.exc_info() - - remote_tb=getattr(ex_value, "_pyroTraceback", None) - local_tb=formatTraceback(ex_type, ex_value, ex_tb, Pyro4.config.DETAILED_TRACEBACK) - if remote_tb: - remote_tb=formatRemoteTraceback(remote_tb) - return local_tb + remote_tb - else: - # hmm. no remote tb info, return just the local tb. - return local_tb - finally: - # clean up cycle to traceback, to allow proper GC - del ex_type, ex_value, ex_tb - - -def formatTraceback(ex_type=None, ex_value=None, ex_tb=None, detailed=False): - """Formats an exception traceback. If you ask for detailed formatting, - the result will contain info on the variables in each stack frame. - You don't have to provide the exception info objects, if you omit them, - this function will obtain them itself using ``sys.exc_info()``.""" - if ex_type is not None and ex_value is None and ex_tb is None: - # possible old (3.x) call syntax where caller is only providing exception object - if type(ex_type) is not type: - raise TypeError("invalid argument: ex_type should be an exception type, or just supply no arguments at all") - if ex_type is None and ex_tb is None: - ex_type, ex_value, ex_tb=sys.exc_info() - if detailed and sys.platform!="cli": # detailed tracebacks don't work in ironpython (most of the local vars are omitted) - def makeStrValue(value): - try: - return repr(value) - except: - try: - return str(value) - except: - return "" - try: - result=["-"*52+"\n"] - result.append(" EXCEPTION %s: %s\n" % (ex_type, ex_value)) - result.append(" Extended stacktrace follows (most recent call last)\n") - skipLocals=True # don't print the locals of the very first stackframe - while ex_tb: - frame=ex_tb.tb_frame - sourceFileName=frame.f_code.co_filename - if "self" in frame.f_locals: - location="%s.%s" % (frame.f_locals["self"].__class__.__name__, frame.f_code.co_name) - else: - location=frame.f_code.co_name - result.append("-"*52+"\n") - result.append("File \"%s\", line %d, in %s\n" % (sourceFileName, ex_tb.tb_lineno, location)) - result.append("Source code:\n") - result.append(" "+linecache.getline(sourceFileName, ex_tb.tb_lineno).strip()+"\n") - if not skipLocals: - names=set() - names.update(getattr(frame.f_code, "co_varnames", ())) - names.update(getattr(frame.f_code, "co_names", ())) - names.update(getattr(frame.f_code, "co_cellvars", ())) - names.update(getattr(frame.f_code, "co_freevars", ())) - result.append("Local values:\n") - for name in sorted(names): - if name in frame.f_locals: - value=frame.f_locals[name] - result.append(" %s = %s\n" % (name, makeStrValue(value))) - if name=="self": - # print the local variables of the class instance - for name, value in vars(value).items(): - result.append(" self.%s = %s\n" % (name, makeStrValue(value))) - skipLocals=False - ex_tb=ex_tb.tb_next - result.append("-"*52+"\n") - result.append(" EXCEPTION %s: %s\n" % (ex_type, ex_value)) - result.append("-"*52+"\n") - return result - except Exception: - return ["-"*52+"\nError building extended traceback!!! :\n", - "".join(traceback.format_exception(*sys.exc_info())) + '-'*52 + '\n', - "Original Exception follows:\n", - "".join(traceback.format_exception(ex_type, ex_value, ex_tb))] - else: - # default traceback format. - return traceback.format_exception(ex_type, ex_value, ex_tb) - - -all_exceptions = {} -if sys.version_info < (3, 0): - import exceptions - for name, t in vars(exceptions).items(): - if type(t) is type and issubclass(t, BaseException): - all_exceptions[name] = t -else: - import builtins - for name, t in vars(builtins).items(): - if type(t) is type and issubclass(t, BaseException): - all_exceptions[name] = t -for name, t in vars(Pyro4.errors).items(): - if type(t) is type and issubclass(t, Pyro4.errors.PyroError): - all_exceptions[name] = t - - -class SerializerBase(object): - """Base class for (de)serializer implementations (which must be thread safe)""" - __custom_class_to_dict_registry = {} - - def serializeData(self, data, compress=False): - """Serialize the given data object, try to compress if told so. - Returns a tuple of the serialized data (bytes) and a bool indicating if it is compressed or not.""" - data=self.dumps(data) - return self.__compressdata(data, compress) - - def deserializeData(self, data, compressed=False): - """Deserializes the given data (bytes). Set compressed to True to decompress the data first.""" - if compressed: - data=zlib.decompress(data) - return self.loads(data) - - def serializeCall(self, obj, method, vargs, kwargs, compress=False): - """Serialize the given method call parameters, try to compress if told so. - Returns a tuple of the serialized data and a bool indicating if it is compressed or not.""" - data=self.dumpsCall(obj, method, vargs, kwargs) - return self.__compressdata(data, compress) - - def deserializeCall(self, data, compressed=False): - """Deserializes the given call data back to (object, method, vargs, kwargs) tuple. - Set compressed to True to decompress the data first.""" - if compressed: - data=zlib.decompress(data) - return self.loadsCall(data) - - def loads(self, data): - raise NotImplementedError("implement in subclass") - - def loadsCall(self, data): - raise NotImplementedError("implement in subclass") - - def dumps(self, data): - raise NotImplementedError("implement in subclass") - - def dumpsCall(self, obj, method, vargs, kwargs): - raise NotImplementedError("implement in subclass") - - def __compressdata(self, data, compress): - if not compress or len(data)<200: - return data, False # don't waste time compressing small messages - compressed=zlib.compress(data) - if len(compressed)") - if "__" in classname: - raise Pyro4.errors.SecurityError("refused to deserialize types with double underscores in their name: "+classname) - if classname.startswith("Pyro4.core."): - if classname=="Pyro4.core.URI": - uri = Pyro4.core.URI.__new__(Pyro4.core.URI) - uri.__setstate__(data["state"]) - return uri - elif classname=="Pyro4.core.Proxy": - proxy = Pyro4.core.Proxy.__new__(Pyro4.core.Proxy) - state = data["state"] - uri = Pyro4.core.URI(state[0]) - oneway = set(state[1]) - timeout = state[2] - proxy.__setstate__((uri, oneway, timeout)) - return proxy - elif classname=="Pyro4.core.Daemon": - return Pyro4.core.Daemon.__new__(Pyro4.core.Daemon) - elif classname.startswith("Pyro4.util."): - if classname=="Pyro4.util.PickleSerializer": - return PickleSerializer() - elif classname=="Pyro4.util.MarshalSerializer": - return MarshalSerializer() - elif classname=="Pyro4.util.JsonSerializer": - return JsonSerializer() - elif classname=="Pyro4.util.SerpentSerializer": - return SerpentSerializer() - elif classname.startswith("Pyro4.errors."): - errortype = getattr(Pyro4.errors, classname.split('.', 2)[2]) - if issubclass(errortype, Pyro4.errors.PyroError): - return SerializerBase.make_exception(errortype, data) - elif classname == "Pyro4.futures._ExceptionWrapper": - ex = SerializerBase.dict_to_class(data["exception"]) - return Pyro4.futures._ExceptionWrapper(ex) - elif classname.startswith("builtins."): - exceptiontype = getattr(builtins, classname.split('.', 1)[1]) - if issubclass(exceptiontype, BaseException): - return SerializerBase.make_exception(exceptiontype, data) - elif classname.startswith("exceptions."): - exceptiontype = getattr(exceptions, classname.split('.', 1)[1]) - if issubclass(exceptiontype, BaseException): - return SerializerBase.make_exception(exceptiontype, data) - elif classname in all_exceptions: - return SerializerBase.make_exception(all_exceptions[classname], data) - # try one of the serializer classes - for serializer in _serializers.values(): - if classname == serializer.__class__.__name__: - return serializer - raise Pyro4.errors.ProtocolError("unsupported serialized class: "+classname) - - @staticmethod - def make_exception(exceptiontype, data): - ex = exceptiontype(*data["args"]) - if "attributes" in data: - # restore custom attributes on the exception object - for attr, value in data["attributes"].items(): - setattr(ex, attr, value) - return ex - - def recreate_classes(self, literal): - t = type(literal) - if t is set: - return set([self.recreate_classes(x) for x in literal]) - if t is list: - return [self.recreate_classes(x) for x in literal] - if t is tuple: - return tuple(self.recreate_classes(x) for x in literal) - if t is dict: - if "__class__" in literal: - return self.dict_to_class(literal) - result = {} - for key, value in literal.items(): - result[key] = self.recreate_classes(value) - return result - return literal - - def __eq__(self, other): - """this equality method is only to support the unit tests of this class""" - return isinstance(other, SerializerBase) and vars(self)==vars(other) - - def __ne__(self, other): - return not self.__eq__(other) - __hash__=object.__hash__ - - -class PickleSerializer(SerializerBase): - """ - A (de)serializer that wraps the Pickle serialization protocol. - It can optionally compress the serialized data, and is thread safe. - """ - serializer_id = Pyro4.message.SERIALIZER_PICKLE - - def dumpsCall(self, obj, method, vargs, kwargs): - return pickle.dumps((obj, method, vargs, kwargs), pickle.HIGHEST_PROTOCOL) - - def dumps(self, data): - return pickle.dumps(data, pickle.HIGHEST_PROTOCOL) - - def loadsCall(self, data): - return pickle.loads(data) - - def loads(self, data): - return pickle.loads(data) - - @classmethod - def register_type_replacement(cls, object_type, replacement_function): - def copyreg_function(obj): - return replacement_function(obj).__reduce__() - try: - copyreg.pickle(object_type, copyreg_function) - except TypeError: - pass - - -class MarshalSerializer(SerializerBase): - """(de)serializer that wraps the marshal serialization protocol.""" - serializer_id = Pyro4.message.SERIALIZER_MARSHAL - - def dumpsCall(self, obj, method, vargs, kwargs): - return marshal.dumps((obj, method, vargs, kwargs)) - - def dumps(self, data): - try: - return marshal.dumps(data) - except (ValueError, TypeError): - return marshal.dumps(self.class_to_dict(data)) - - def loadsCall(self, data): - obj, method, vargs, kwargs = marshal.loads(data) - vargs = self.recreate_classes(vargs) - kwargs = self.recreate_classes(kwargs) - return obj, method, vargs, kwargs - - def loads(self, data): - return self.recreate_classes(marshal.loads(data)) - - @classmethod - def register_type_replacement(cls, object_type, replacement_function): - pass # marshal serializer doesn't support per-type hooks - - -class SerpentSerializer(SerializerBase): - """(de)serializer that wraps the serpent serialization protocol.""" - serializer_id = Pyro4.message.SERIALIZER_SERPENT - - def dumpsCall(self, obj, method, vargs, kwargs): - return serpent.dumps((obj, method, vargs, kwargs)) - - def dumps(self, data): - return serpent.dumps(data) - - def loadsCall(self, data): - obj, method, vargs, kwargs = serpent.loads(data) - vargs = self.recreate_classes(vargs) - kwargs = self.recreate_classes(kwargs) - return obj, method, vargs, kwargs - - def loads(self, data): - return self.recreate_classes(serpent.loads(data)) - - @classmethod - def register_type_replacement(cls, object_type, replacement_function): - def custom_serializer(object, serpent_serializer, outputstream, indentlevel): - replaced = replacement_function(object) - if replaced is object: - serpent_serializer.ser_default_class(replaced, outputstream, indentlevel) - else: - serpent_serializer._serialize(replaced, outputstream, indentlevel) - serpent.register_class(object_type, custom_serializer) - - -class JsonSerializer(SerializerBase): - """(de)serializer that wraps the json serialization protocol.""" - serializer_id = Pyro4.message.SERIALIZER_JSON - - __type_replacements = {} - def dumpsCall(self, obj, method, vargs, kwargs): - data = {"object": obj, "method": method, "params": vargs, "kwargs": kwargs} - data = json.dumps(data, ensure_ascii=False, default=self.default) - return data.encode("utf-8") - def dumps(self, data): - data = json.dumps(data, ensure_ascii=False, default=self.default) - return data.encode("utf-8") - def loadsCall(self, data): - data=data.decode("utf-8") - data = json.loads(data) - vargs = self.recreate_classes(data["params"]) - kwargs = self.recreate_classes(data["kwargs"]) - return data["object"], data["method"], vargs, kwargs - def loads(self, data): - data=data.decode("utf-8") - return self.recreate_classes(json.loads(data)) - def default(self, obj): - replacer = self.__type_replacements.get(type(obj), None) - if replacer: - obj = replacer(obj) - return self.class_to_dict(obj) - @classmethod - def register_type_replacement(cls, object_type, replacement_function): - cls.__type_replacements[object_type] = replacement_function - - -"""The various serializers that are supported""" -_serializers = {} -_serializers_by_id = {} -def get_serializer(name): - try: - return _serializers[name] - except KeyError: - raise Pyro4.errors.ProtocolError("serializer '%s' is unknown or not available" % name) - -def get_serializer_by_id(sid): - try: - return _serializers_by_id[sid] - except KeyError: - raise Pyro4.errors.ProtocolError("no serializer available for id %d" % sid) - -# determine the serializers that are supported -try: - import cPickle as pickle -except ImportError: - import pickle -assert pickle.HIGHEST_PROTOCOL>=2, "pickle needs to support protocol 2 or higher" -_ser = PickleSerializer() -_serializers["pickle"] = _ser -_serializers_by_id[_ser.serializer_id] = _ser -import marshal -_ser = MarshalSerializer() -_serializers["marshal"] = _ser -_serializers_by_id[_ser.serializer_id] = _ser -try: - import json - _ser = JsonSerializer() - _serializers["json"] = _ser - _serializers_by_id[_ser.serializer_id] = _ser -except ImportError: - pass -try: - import serpent - if '-' in serpent.__version__: - ver = serpent.__version__.split('-', 1)[0] - else: - ver = serpent.__version__ - ver = tuple(map(int, ver.split("."))) - if ver<(1,3): - raise RuntimeError("requires serpent 1.3 or better") - _ser = SerpentSerializer() - _serializers["serpent"] = _ser - _serializers_by_id[_ser.serializer_id] = _ser -except ImportError: - #warnings.warn("serpent serializer not available", RuntimeWarning) - pass -del _ser - - -def resolveDottedAttribute(obj, attr, allowDotted): - """ - Resolves a dotted attribute name to an object. Raises - an AttributeError if any attribute in the chain starts with a '``_``'. - If the optional allowDotted argument is false, dots are not - supported and this function operates similar to ``getattr(obj, attr)``. - """ - if allowDotted: - attrs = attr.split('.') - for i in attrs: - if i.startswith('_'): - raise AttributeError('attempt to access private attribute "%s"' % i) - else: - obj = getattr(obj, i) - return obj - else: - return getattr(obj, attr) - - -def excepthook(ex_type, ex_value, ex_tb): - """An exception hook you can use for ``sys.excepthook``, to automatically print remote Pyro tracebacks""" - traceback = "".join(getPyroTraceback(ex_type, ex_value, ex_tb)) - sys.stderr.write(traceback) - - -def fixIronPythonExceptionForPickle(exceptionObject, addAttributes): - """ - Function to hack around a bug in IronPython where it doesn't pickle - exception attributes. We piggyback them into the exception's args. - Bug report is at http://ironpython.codeplex.com/workitem/30805 - """ - if hasattr(exceptionObject, "args"): - if addAttributes: - # piggyback the attributes on the exception args instead. - ironpythonArgs = vars(exceptionObject) - ironpythonArgs["__ironpythonargs__"] = True - exceptionObject.args += (ironpythonArgs,) - else: - # check if there is a piggybacked object in the args - # if there is, extract the exception attributes from it. - if len(exceptionObject.args) > 0: - piggyback = exceptionObject.args[-1] - if type(piggyback) is dict and piggyback.get("__ironpythonargs__"): - del piggyback["__ironpythonargs__"] - exceptionObject.args = exceptionObject.args[:-1] - exceptionObject.__dict__.update(piggyback) diff --git a/contrib/site-packages/Pyro4/util.pyo b/contrib/site-packages/Pyro4/util.pyo deleted file mode 100644 index 7272c1a545507ce0c835b06b14f0a4fa600fc3da..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 26310 zcmds@l``()&CA;lrS5!mk-MR1H`|gi(&pqedbI!e`KixU-h5z)8u%GyQ%){!-1~&4#lIATNre;lDkEhcB*gb{~mXn&g{-! zi!P$|ZSFR`+Ux3!WuJ=*ZehQxJ>nLfYoE`WKj5YgxV7JQ(Lom-P_+w(+%4xWTs!OD zXQ+qW)B`RWa?u0IBeK}%+O7~isFdi4yH&JT;xusBonvJDpKM;shH%tP9d&EtR{zoe z%lfA`*3WCF{^`QXbYXIB(kkqC3lDYI`RXV4`Qxlk`)1)`>p6*?nm;bAd|a5U^||N~ zH_Kf2yW38n`21AhY6C6`EOkeodd$@Zv(%k=>Z7i^wi7xDkNs~vFX6njGN3#Z(< z6vIxtsnafcbYt0ro64>~oz38gn;LP^;~R?~+Eje9xi`n|(Gq=Af7X9BtK*EDI^$}G zU3A7RJna^HUHgkAA76Mz={3*`T;A@6rk-_8@Z8D~kDhbUXI%8Wi=K5VI;82QsPLcM zXI(8YeEwc3dcoBmRQg8+*K{s=Y-5Lyd`gG^d)DFio^rLL2K*OY?IDZ2!;b9b8#Y=3C7mjFwtS6kMKJ zh^k3E5(J~I<@I)bZaxWS)`N@frKlOyqTsDob1oRJ>-E)k<%3rHdc7HqG^6C?$Uyyn z@i!h1(5&kI%!pd6mU!*u>EjvYiT@)3a|{;UEV&Q<(0#~jk6Y|_?W^uXFq$~4mpGu# z;z>cp02>isGBKj!0l9KjuuSXW+uL2=-gXJps7GLazvOPxXu;hqs(iA|nu9cwOID9d zdQ}O1py_10Tj+BO{q9zeyGh9&*Z!gVu%rl$QusqlJUqeg#3WUG(YY@c+!u?G$R1`u zWI8StoC~95wcU(^O3e=wg>FJ<8YqhPnR)bnAXtt6dUR_@1Lye^6aVQ|S z9Y>AXkt$$tJ?S$czTjcP2g)NCo|;+^AlT{b`-BRl0;>cXf;^w zFe%Q0I3jkYwVa%3w3y*Dm+CX^N_+jx>+Q-?bfeW?jL&#FFfyB^ocUbTjzP8f%(Eje zo@vx)&MX_H&|6$jqSMRZYGp3cmn`-(TI?efIY>($e?63Pj2j{&K0x3K{e_){-RZNl zaFAI1dD836>y<{m7F61Es}PvvER#`A)|aCoo^P!-YQantRGM8gq`Xr>s~s$?vb0hZ zY_{?WK$cfF4@ON7sjrhzj$fvLiuWVest`$^P_JY4qAoj zP}7#mKw{NTr7~thwP}6JF~CaNbdtX%Q@zK-K|{?4uL+Ki61Z*sJ4%Jei~9=u3J(+> z;P*(Ow=h`fEA^GCg7!|HL7q{b_}c&)A^?XKd4JO=GjWJHan^Mf-^>W!bYcWBTjQuK z^^_6#@@rk+u6-gSfQYnzxg<8RO@wFBUTYC?LQ2 zSJH=&pci&#rFU5yH~U?4u#?1V+L2EGy1KE$eb}e*FYI<7_Pg2AhCU8@D(E}B({sGVx;fw$4mj4j9}U8!WF8av3!C#0_mOz+o`?94 zbbj}|y^?aWJl|3J1VLZ>rb`Z~3%p#o_LtpFM#}#jkk1!=oW!AryT_rsZ(^C)YIi#0&oBPvhx-%zOg%iUMX}LZKzkA(;6iDm_ik!(8Bf-U4NdQ45 zUX+l7@Q6s!fSr13nIHn4G9f}DgJ)sEje4UIKrbbHm>d%z0xAD`rCqPgG$M&JK~$;E zBScA5GH&H1o6nmp(mas_^Ofs}Q>|dR-MU^!W>p0pt}YoS6H!hf$XF*^OZ7zMma;A( z2+XcFt0se4ue?NLWrBJlu_joJ8AEWVbTsl)W@eB|S8dd*k~#F~-K47LiLCE(iIGwk z$HZha&>miCnE(v}J(KS1nPzS1`FOaj~_jS#NUj9<2BH%X>Ez+O&}Fhd$m=I&JLKcDDghoK~}W|+iy1! z$-Kb9yMIYDwUMJhoE4sJN0Hub_?INuu}}a>OvM6|Dn1?uFI(G^%aou$lgjfPP2+%x zJHgCqy-^cZM;Sm27TT5^J9aEMI{EWTQsj%8nLgH7Mpgi0_F9rMev(ItpjUGY|@ZB7D|okASL2u~0ML znbLc=tq8UG`5ER+R4|%|;69D|2T}XAN*q;{u0J8^ zYdrCf2-sjr`$||eOe6zI zsK&lD@42BDG%M|)dT$DXP=GM%pd?%7S(&X^i-~AY4w4IW>1#^pA#kPLg?t{hK5QP3 z6G6UgKOoV>PoJO7rk`I_H8#KLY)TbyA<7$&qAUu!CNzcqNH7t>kW*j*V@C zvD8Ga*vmG%s;HQo5X>O2j^-~+3X^YYJf zMM|Os{m@TzRsEQ&Nj%$D((i!&Z+f#uD*D3A2=9p*?sFQ&a z#ty~&vDaYumO%bCftR|p>NNCCD=&P95?eAUo7S89_D%ZY*IUWfn`002nxtQ_GsAdwmeQc<_W!M>BqHDj-6Kklh zP|ZfH-#^=^!%3>RABOl@`th4Qe~yPZ#_;|@*AT@c^!%h|QFVDly`tF9D+m=xHL(>7 zIJ`{5|A;4>RPh2aem#b_UmyZg9E^udXJU(%rMrlk5GMu+H74kNFNNqhc%axwtj z!B7G+SwtQ{^20dV0oKcG8aBZz)I#=bL=XtedgbxKy;D*o576dn^X>!wm*jbouE-uq zL(V9$;lzJs`q!RGhk4=J6EYe|X`AhI+ZbIuz7*Y%Cp!}>18x%H9_dM^#;D9&qn!+%_6DdZb{Vy=0@yDx}W;W9#~CviVo2{*YDwUsQcMR7`YK{rv(u z9yt;`Zhbn8y$7oeKzu;&^7P1dOy^cUDwqSo4%x}>I#@N=0eAl=QL^+EufeU#Q?`(lq^p zwgRSE9|@nLD%%#z3|^mEMImAYr(BL3tpuHG{3s=AEp=tF*}4(T^DvUEjZA42pP|@D z@CFJSsZFFd9X=_233esCAKI;Dly7(@mCKmiQO_q)xja5B;%m#^x=_awLl#5ys^w?Z zH?dOz%{F|tYq2DG;e^Jdv|Em3<&RCHai!f#TGdv=W5cf5KAvA(!5|yZ$ z>0fOa0`iv7IPFz8Q1X98LVS+EmG&SN94tIsc!FQFu*E&Z_Y`{2sqR6$+5_QJd>^UO zw(dr0u?OAj!{k)O{WN|N#ZjJkUkM&y6c5@Ze<19zMx8)vqiJBg?rhiyW zYlZscO&cAbX#ScxJ5l#eeb~Nbquphny&-CMiyf$H+HFZw1rRq&+G9ye1yC_d+MBfp z!RN!~n_2B%grcIp%*M3qcOaKT_SnjyNkPp0^EZtxcX?DYSDTQFwYEcKS7p@&O zCkl8BeiVm?&aE~@^qa3BpNR0q7x-yMv#YUJ)OIN{19@@9*#47w zoC#OO_g1mRw&Q9GJ-sv~bg_>9oF1PIVi5^*t{PGPh}k~7-ai{&G=guOA{L$R%7nzP zt;6%t#E9P1)wfvEcvl(E?Yf#V$F^Afz?ZW3EA{GPBbvw!9maaQ^521t%~u*-1;nws z3cMA!nq9d^H|CNfj^0&3WY>T0S=LAxkzytipFF#YpDB*L<(wl;w&gw8Hm;l-4sCX1 z=EYPIkJvCfc_Z<|REivcGpAWHJKP%8uS}_DgYjYUb=#uBkd2du4W+1u6A>@n znXzW(p`CcoMloc~nSHBy5@*FI8{NPr>-i&ima=%8M4PYBxVe~hDu~#mZi0WJS-b=2 z#$-1_+6vwQUSqhSa3Yl_GUc2BPXP8DT3BqA$^)TejkU4cevI9X!O6y!PKh;n=d|=u zS2hu0D{Hzl8P*O;$4)b`hf>_ZI@4==o4*MwUf@`wvSF0=bv>ys;V|wE)P`_cq_&nZTWhBCep-giD4lL?gq z_3lFe88R{C2*%+a-j1NIIE28VQ13@AcRR7{ky7JP1epD#?Wg4K!XrE~82jJB!hzy` z#FxIi?BA}^D5M^0jkq%ORrVF|BioxvSh-?*JM!4PEx?034|pG5*+uRq3**vjeM7+JM_}56aDGy~B@&dc0RuPvinUh~s!>lSh z6W|aB2a69O@Kr^3wfKdD8ejaU1i3oR%VO|AxqrSQNdTD`z+Sj^Q1D`+<=kz8(iiZo zdsHlZ0$73C`eNbJi~qbY>rWPZO;DnjI!RM-zu=ppx~#@+R^yMlX7YEeedIP@NE^r= zw{}|Z$YwUNkC@NkHrd$HzV@m@1VCiGb1Of<6A&LeK1e6GD!L<(2o%rp7!?MYt{srb zt2(S>)bY8(3gWJe3i>A17Kt9hjZFyf)F+A!O>Di6JPiQRE!Y>PUNls=cUloI56ne4 zyezk0c}eJ?!S?#?)o>s2&aN%|n|fVTz@S|yW^GG56v}rQ_Jlq-$w%!-~b}yFjo8)#z6n;amJoj8x=m*n8N$`3;n8L7W9Zu1maG4+#kZP?ess!mcz1iTR zV&e+Lbi&UmxTHW3I zc0@o9TextM!s$y~wakl)RXR0@>C29H#9yOY+pe7tHn~UH5wIJ`>E6l3?<#Vk+kB?v z#%y{izcsrkA5=Dab_@*Za6R{cv4x!rVct{elx^`?-Cd4WzMiVqa`xD|u@%m`fm3yf zBdC(W5g0B+C|gJ+J~dQWHYwat&b$=gy!ghOV-r`)SHjCzFOOcngd_Iy-e zXEv28;Kw5haz6}3MC{!^A43sH-|HHRaIA+%%TDq5X9>EcPo&#*j@4w?CmFY>bLz5@ za4-oQ$qVw`-{`hqr3I%)b1CdKtp_CBOAGM9Tbl&N0Sc8d9oAN>@EofK1Gr-_wZYaY2Y?6eEwrjQ+oBpSa|XLrHfNzVfo6%(YG&+ z*^WCW?d@{C<~jVBsxYoER13eYKn%V^gYNM}8ee@|W0w@x+rxpsV#bHVk4UQW*q@N~ zHJ+aq&wZJ?Kb_ewc@^C!aoal?q%?S)wVCbLVUU~Xsh>|@yHV!*lnruW$Q^7@0^X>m zMRT*hw=-UiakBZO5h6BN_{E4Ey)1H+!UU>ib{I;kus>Kt6KqpO@WOwnKp1N(Hh*?U z);E6;gi7uU_M;BA1eiV|I#RS31(0nr1R7Cq1CM0z92Xew`?%C*+8iAaHIa#JK3V7o z(pNiruG|cF&rXFFl4NQ1Y>u^=xT*m_^%o$OK4&P(; z!%Y;Kcg-iVo$$BmwH-^J=Pq-*LwskxPo-I4hPl?B*wcpP?<`;8B>?pZ3Ew+k2byM;&rToZLzC#1IWKD@H zd}Y&6?hQXMF#LbP4}MAT@m%vKzz_0q-!J^&>s`|?ULY2rUp?_-1e-Q;xpa{3;*4Dg zmB*|4R3ednZNnz?`f|2mle}OegNzv?W>_GL6q6cc9^!Pq%njb8kf&M$bP{<>&Dc%? z3Q?{_vlZ@#N$rxh_sh2uUNZZ;D!z{(B`D=Qzik#r41vq57-q3BESW-)!){QuMsz)D zm>w{7e64NW22$qVs&|m>bpEe*FjCt0Lq(5ELM=0{ZgkYV>rV1_DeEbYAZxqJd(6sz zMS~Egu#@cle`DtG^($lJSMO_Pew%?I5N=}TC1d8d$g67P`V&sS#uICYAE9o&o01lD zku-jFO;5 zYR5yZUDK<2g{)5I#dhSvR&InVM>{H4;%dEa+tQR6*p&sV$?WNui~^XcSmQB<>#Ypm zqhhNQ&ruHBckM*(vI*I`!%4Dq{F?Abkan2wAMOYmIq;v-STd~|*dfqDH%alJHC?aV z%k&CpwcN(T-y;maN3aQ6;rEpxn%#j^_|JL0A6S*T^&2_K$e^JC=^5p*1tk8#7RI*f z*qFPwXXzXuEm#AI#M;~xuj znOxi}JxK<-9DI^qw0;bNz;5$_pzvShgWIU-_M!=yh;9V;L|Jbd=HJzzgv72myqnNF zo-Hf(Ed_fNn1VsMT?&S`CN7VAfkb1vtBT=!%J{Z|t<`1Uq6^>9I3*g(wmgW#I@V(o zPWrc1g|d3K@r?4A%&6nI>;D4Eq4BOw1yzgblFEd;s&kN@2nFSzWiV9Mq4*7r2R?nFxA?f)xA%r)Kl*|&rLAam5!UWuD&oYtn5T9mEzilhgJ~R zmssg-d2mfnGw0^+i4P909Sr&8^tA7b%_qc0IIJ~JX0)`7lNW%fR(0PE=cumRF@YRX zx}{H+Z#;f7b}jfmmJe2g2%GLRe&A0Cj~69`5I}!Tk?L zWQK6JT|8I!&F;r}my<>%z7ZbhWcv&+P-KJ`;3sS(1Z8b6fJL6o0_-M1gkw_1yKpI{X%1l3FVOBO=xXk3BCQk1MmcM!DKIk}n zoOV(pnzp5(j+xrcX(m9&35O-xHXS$c>yG?{O0U$p8Z7)-Ubke1BrR$%ry@bL;-j7Ot(NMj8?^%0Dd z5tky*C&}{JGiU1b!e8smX_^`*>!URyOJ{T?wC=4t^W4)fJ^j2@VJ_&VB`lXapQ#Dn z8MgD+zuUFSAwcoYqjs}__g;TjX+j6@)GP5lbt1Hr6XozZQvAM9GGjiF%XXc(xjKzRnGF{#EqZ(uR!!e2Z6qcj8;bOtaHy&2b;&oUQawBrW_cWywc|OJ$gO zev;t)D9~CD|F!}{hrg&;OTk}KAkiZH1qG(tx}eyX6nv!M zwt`y9 zUg%$OcWK}8gQH@JgPdwP#)%gHyBF2jiNfB}q2D|7(4ixT9zGOk$AcPnd-{E`A3WX3 hSN>O0yJCAu)1h8nx_Fc_4_hqFDUws_-{0T+zX7FOCaVAd diff --git a/contrib/site-packages/Pyro4/utils/__init__.py b/contrib/site-packages/Pyro4/utils/__init__.py deleted file mode 100644 index 3c55dacc..00000000 --- a/contrib/site-packages/Pyro4/utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# just to make this a package. diff --git a/contrib/site-packages/Pyro4/utils/__init__.pyo b/contrib/site-packages/Pyro4/utils/__init__.pyo deleted file mode 100644 index 6b1df4e9eb29da1927bce9ca98fcbf5b412134d3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 198 zcmZSn%*z$`cwta70~9a#hLke#rj5i=K49AN%{p~6(C-5 zW=X1UL1J=tVtOh_XHmY1erZW&PO*M`d}dx|NqoE>P%#J4PMh5Pl+v73JCMtYftUdR DnbI@~ diff --git a/contrib/site-packages/Pyro4/utils/flame.py b/contrib/site-packages/Pyro4/utils/flame.py deleted file mode 100644 index 9672652f..00000000 --- a/contrib/site-packages/Pyro4/utils/flame.py +++ /dev/null @@ -1,299 +0,0 @@ -""" -Pyro FLAME: Foreign Location Automatic Module Exposer. -Easy but potentially very dangerous way of exposing remote modules and builtins. -Flame requires the pickle serializer to be used. - -Pyro - Python Remote Objects. Copyright by Irmen de Jong (irmen@razorvine.net). -""" - -from __future__ import with_statement -import sys -import types -import code -import Pyro4.core -import Pyro4.util -import Pyro4.constants -import Pyro4.errors -try: - import importlib -except ImportError: - importlib = None -try: - import builtins -except ImportError: - import __builtin__ as builtins -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO - -__all__ = ["connect", "start", "createModule", "Flame"] - - -# Exec is a statement in Py2, a function in Py3 -# Workaround as written by Ned Batchelder on his blog. -if sys.version_info > (3, 0): - def exec_function(source, filename, global_map): - source=fixExecSourceNewlines(source) - exec(compile(source, filename, "exec"), global_map) -else: - # OK, this is pretty gross. In Py2, exec was a statement, but that will - # be a syntax error if we try to put it in a Py3 file, even if it isn't - # executed. So hide it inside an evaluated string literal instead. - eval(compile("""\ -def exec_function(source, filename, global_map): - source=fixExecSourceNewlines(source) - exec compile(source, filename, "exec") in global_map -""", - "", "exec" - )) - -def fixExecSourceNewlines(source): - if sys.version_info < (2,7) or sys.version_info[:2] in ((3,0), (3,1)): - # for python versions prior to 2.7 (and 3.0/3.1), compile is kinda picky. - # it needs unix type newlines and a trailing newline to work correctly. - source = source.replace("\r\n", "\n") - source = source.rstrip() + "\n" - # remove trailing whitespace that might cause IndentationErrors - source = source.rstrip() - return source - - -class FlameModule(object): - """Proxy to a remote module.""" - def __init__(self, flameserver, module): - # store a proxy to the flameserver regardless of autoproxy setting - self.flameserver = Pyro4.core.Proxy(flameserver._pyroDaemon.uriFor(flameserver)) - self.module = module - - def __getattr__(self, item): - if item in ("__getnewargs__", "__getinitargs__"): - raise AttributeError(item) - return Pyro4.core._RemoteMethod(self.__invoke, "%s.%s" % (self.module, item)) - - def __getstate__(self): - return self.__dict__ - - def __setstate__(self, args): - self.__dict__ = args - - def __invoke(self, module, args, kwargs): - return self.flameserver._invokeModule(module, args, kwargs) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.flameserver._pyroRelease() - - def __repr__(self): - return "<%s.%s at 0x%x, module '%s' at %s>" % (self.__class__.__module__, self.__class__.__name__, - id(self), self.module, self.flameserver._pyroUri.location) - - -class FlameBuiltin(object): - """Proxy to a remote builtin function.""" - def __init__(self, flameserver, builtin): - # store a proxy to the flameserver regardless of autoproxy setting - self.flameserver = Pyro4.core.Proxy(flameserver._pyroDaemon.uriFor(flameserver)) - self.builtin = builtin - - def __call__(self, *args, **kwargs): - return self.flameserver._invokeBuiltin(self.builtin, args, kwargs) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.flameserver._pyroRelease() - - def __repr__(self): - return "<%s.%s at 0x%x, builtin '%s' at %s>" % (self.__class__.__module__, self.__class__.__name__, - id(self), self.builtin, self.flameserver._pyroUri.location) - - -class RemoteInteractiveConsole(object): - """Proxy to a remote interactive console.""" - - class LineSendingConsole(code.InteractiveConsole): - """makes sure the lines are sent to the remote console""" - def __init__(self, remoteconsole): - code.InteractiveConsole.__init__(self, filename="") - self.remoteconsole = remoteconsole - - def push(self, line): - output, more = self.remoteconsole.push_and_get_output(line) - if output: - sys.stdout.write(output) - return more - - def __init__(self, remoteconsoleuri): - # store a proxy to the console regardless of autoproxy setting - self.remoteconsole = Pyro4.core.Proxy(remoteconsoleuri) - - def interact(self): - console = self.LineSendingConsole(self.remoteconsole) - console.interact(banner=self.remoteconsole.get_banner()) - print("(Remote session ended)") - - def close(self): - self.remoteconsole.terminate() - self.remoteconsole._pyroRelease() - - def __repr__(self): - return "<%s.%s at 0x%x, for %s>" % (self.__class__.__module__, self.__class__.__name__, - id(self), self.remoteconsole._pyroUri.location) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.close() - - -class InteractiveConsole(code.InteractiveConsole): - """Interactive console wrapper that saves output written to stdout so it can be returned as value""" - def push_and_get_output(self, line): - output, more = "", False - stdout_save = sys.stdout - try: - sys.stdout = StringIO() - more = self.push(line) - output = sys.stdout.getvalue() - sys.stdout.close() - finally: - sys.stdout = stdout_save - return output, more - - def get_banner(self): - return self.banner # custom banner string, set by Pyro daemon - - def write(self, data): - sys.stdout.write(data) # stdout instead of stderr - - def terminate(self): - self._pyroDaemon.unregister(self) - self.resetbuffer() - - -class Flame(object): - """ - The actual FLAME server logic. - Usually created by using :py:meth:`Pyro4.core.Daemon.startFlame`. - Be *very* cautious before starting this: it allows the clients full access to everything on your system. - """ - def __init__(self): - if "pickle" not in Pyro4.config.SERIALIZERS_ACCEPTED: - raise RuntimeError("flame requires the pickle serializer to be enabled") - - def module(self, name): - """import a module on the server given by the module name and returns a proxy to it""" - if importlib: - importlib.import_module(name) - else: - __import__(name) - return FlameModule(self, name) - - def builtin(self, name): - """returns a proxy to the given builtin on the server""" - return FlameBuiltin(self, name) - - def execute(self, code): - """execute a piece of code""" - exec_function(code, "", globals()) - - def evaluate(self, expression): - """evaluate an expression and return its result""" - return eval(expression) - - def sendmodule(self, modulename, modulesource): - """ - Send the source of a module to the server and make the server load it. - Note that you still have to actually ``import`` it on the server to access it. - Sending a module again will replace the previous one with the new. - """ - createModule(modulename, modulesource) - - def getmodule(self, modulename): - """obtain the source code from a module on the server""" - import inspect - module = __import__(modulename, globals={}, locals={}) - return inspect.getsource(module) - - def sendfile(self, filename, filedata): - """store a new file on the server""" - import os, stat - with open(filename, "wb") as targetfile: - os.chmod(filename, stat.S_IRUSR | stat.S_IWUSR) # readable/writable by owner only - targetfile.write(filedata) - - def getfile(self, filename): - """read any accessible file from the server""" - with open(filename, "rb") as file: - return file.read() - - def console(self): - """get a proxy for a remote interactive console session""" - console = InteractiveConsole(filename="") - uri = self._pyroDaemon.register(console) - console.banner = "Python %s on %s\n(Remote console on %s)" % (sys.version, sys.platform, uri.location) - return RemoteInteractiveConsole(uri) - - def _invokeBuiltin(self, builtin, args, kwargs): - return getattr(builtins, builtin)(*args, **kwargs) - - def _invokeModule(self, dottedname, args, kwargs): - # dottedname is something like "os.path.walk" so strip off the module name - modulename, dottedname = dottedname.split('.', 1) - module = sys.modules[modulename] - # we override the DOTTEDNAMES setting here because this safeguard makes no sense - # with the Flame server (if enabled it already allows full access to anything): - method = Pyro4.util.resolveDottedAttribute(module, dottedname, True) - return method(*args, **kwargs) - - -def createModule(name, source, filename="", namespace=None): - """ - Utility function to create a new module with the given name (dotted notation allowed), directly from the source string. - Adds it to sys.modules, and returns the new module object. - If you provide a namespace dict (such as ``globals()``), it will import the module into that namespace too. - """ - path = "" - components = name.split('.') - module = types.ModuleType("pyro-flame-module-context") - for component in components: - # build the module hierarchy. - path += '.' + component - real_path = path[1:] - if real_path in sys.modules: - # use already loaded modules instead of overwriting them - module = sys.modules[real_path] - else: - setattr(module, component, types.ModuleType(real_path)) - module = getattr(module, component) - sys.modules[real_path] = module - exec_function(source, filename, module.__dict__) - if namespace is not None: - namespace[components[0]] = __import__(name) - return module - - -def start(daemon): - """ - Create and register a Flame server in the given daemon. - Be *very* cautious before starting this: it allows the clients full access to everything on your system. - """ - if Pyro4.config.FLAME_ENABLED: - return daemon.register(Flame(), Pyro4.constants.FLAME_NAME) - else: - raise Pyro4.errors.SecurityError("Flame is disabled in the server configuration") - - -def connect(location): - """ - Connect to a Flame server on the given location, for instance localhost:9999 or ./u:unixsock - This is just a convenience function to creates an appropriate Pyro proxy. - """ - proxy = Pyro4.core.Proxy("PYRO:%s@%s" % (Pyro4.constants.FLAME_NAME, location)) - proxy._pyroBind() - return proxy diff --git a/contrib/site-packages/Pyro4/utils/flame.pyo b/contrib/site-packages/Pyro4/utils/flame.pyo deleted file mode 100644 index 218c6f1f70bf0fae97d705f672284a84c0f9ab08..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17806 zcmd5^O>7)lUVqiyZrdH_b23gmA3HT4+3CzC?b&5u*lafB%V9Dko=~2d%$S)ix4X)A z#qF-%deydjNFdq_2nmTr8i@nq!VL+D3tTvGLL4}7;Dito9FVwh195@F^85Y&_0e_$ zT8&n2JFe&VUcIV%@BjV&|Nl$%KaVzk@UMUDbxig%fxlND8)N_3F(xo(8Kq-_is?FL zt74Wb#$mf^x>d7Wm3yey%yKQSPnhM2yk0lUlX-n=q&_`TZ?HZ*kq-ZTHahU-4kZ(q**>`ocq03 zi)a2Pe+C0i6IRUwyzFJOSu@FbCpcoh1bm)1y;&3D_lyZ< zOmNf$$IO$edEx+2$7RG>6Pz&NoLR%nPO|5TF`q8KV1iS5?{h`(izYao_dZ|rzGMP? zV8T3jSzdCcxbZS)m%aLI(fbV(%n?Pw3+79p>=hHd$a~AK30{)QtGv6^{04Asj{?t) zd&4AlZ{4|ecj1!j-inhjTJO1c;!Zn_;+}hLkj7i6cHFyhFzANv!frpdVbW?WwC&Jc z9i(nQPQzXrwY%M+yB#J&H)!|P!z3P9cc(pc<25(rGf{8dO~Neqt%gqhqI++cZeXgO zzWjsL2Vp0*E!Vvf_lHTezLC1CL-%&F753a9bl;CLk!Hl^)ujDJoNPzEu+i{;sDqFeo_8#ejXw2s2cn3OA8!;K!`pEl4~ z@5Za`uD{jpH_cfK2~&i4gVY}qIcFZSD=&Gvf8CzJS{`9T_W?fe?apoNh~@j))lUmNr~B*r(; zVYy)A%sBQd{0%}9*T|5jF6n}sujhhWF6R8DhKnB^^8IJRDSQ6B4CI^LPOLzC`o*vD z{%hyms5kPBhCPmXUEVXPD;|N}AzASvo!oegrF@P`YRm&*c*<;|qnS4y;|9|vXb|I@ zb(7pBB|HH3MJhe1kUZD}Spy(UnlotkM!NL&Qu7GDuo#h2B2sEeeGw;OdE;oqvfdB^ zDu-&5lp@qR(&XqSv5@kv9aRJh%E{G-F7DwI!i40r7!H`P4jVFW1=dL zaQHb~mV|DcX7!ZwTxG^d2`XVxW!Vnz@`uY6_WT$Xu;((cg%Y@MM4m#_R-^(YSd|Kt zM@=d~)`V1`Na|9VBC@{3<3R3|R2s%xY7!=P4#mAB-W^iLwfBn3Rv|42b0MAMyc;#<;*xeQ+d_tkn(vI zym$o-irSj++lvIP!tzHBIV%N#D}*t*PB6_10PK)OXTgz6O}=YU^oH-RhiNa|X(#K} z_ftv)>EcYZ&vNiN+d5~3vn;jE-lx2m(AA{ikXK$y!9vis;X;zciFXS9-gCT3De0@~ zyBk6e2I*<`jF2)RupMuPiVm*YBdrPMA%(Zbi6L^Gth0?uPYn(8c!Yp#mx`%p>T&^jsN)}OByEps?Xlv>C4Vc3S?yd^a9 zCFJPiGGqxSQ7KUP1nUdR^_18XlBe46!fx2MVTnB9&v3Wnr^9}j^1Ezox4Q#vr3oyB zRTzWgM`*Z9WkFq^LnB8f$xfj0^Qb`i7XZmB6&ggStMa8T77g*BZXQgUwFxtune3Jf zeOY9`+fLoLchBu!$aX~To9FDC>^x_$G)Z6rE8p*Q+m?!iCX4T@zC}|>kXAby#6uJ) zn+P+>U83WoB$A=D+i6B04(ij%!%774eTunfhW+!77_Hv7(dE63LX1h@58{sRdoQr} zRTi(aILG1@7Ux;eB=U%zCa#gvv6w|eU)SAl2gBrt z#CH8L9{e6E;n%e`VN#M>ErIm3Ntx}q-F#d1vo=%86!lBk>!v7m1Gyi%j%zKN7;Im4U-(u|oi;KL0 zON)c2Op3NmT>CNeGgC^hub>nxNHe4G%vhDB&+--3; z?BJgKP|}D(`5cvf3WZMXns}APgYkr}X*e&`ivgd3dNJUZGJ8(`eg}ngY)crFrRH6Iy%)f+Y9C5rr-*KIml(*I1 zg!9-Al2AOz;{9~nXjxdgbgj^@ocW=%sTAG-a=ys9*HM8702>xMSH#?dm*)wB5!p5k zAEyb-uP%Bt%hc_s@6BF&MZIKvX(tXsS?|-B63UaE&z^_N9iR70c)>Vj4_RZEOXNz7 zGpid|HfVDeCUL11#R2shoRs(#zjg9$1N$`dP!W#C0Y_pNm~V?heH% z`-Cw+1Tib-m){EtnIR;)+D70gu{4dFo(8XMXz50GAriyjyaJABQ05@*%@NaYFDmCi~mCGJoDFXYLvPJcc0^iUc7TBWplN>~%8`?g|9D51{V0q8{RF zYO#^Zgp9_xUXe(hE@Gp4kyYwyyX}Mfvhug)G@X z1F{7Ax1z?D#;G3F%opFII}#L#9$#$OB0310EDL8%Z~z|6nspJn_XE&xEqm@0_0}WC zl_wnFi>8br-fDMkD8`w04L3ZYaT+h;Q-J*@6BqhWV@I4GH#qbriv-?)1kFuoTJXPQEQA%^JQ=~cNu5x=Wssse@q5E)be;IiC6I{8x zq7jo{vVzm-*k{0v`uD52J%ZrtXpYz#lzc%uZI2UYoYJ4-%2!AP@Zu_Eg?o$)AdCAC z;_TT*2HllnTn&0jxE@*98bYDSNea`|!P;7w91hIh!9@QISEfZzIKr0uW%gB!@JIw6 z;pFk55?N9^crK#~`9oZigCQG>8M?+h=#n2|Pvr3zsDxkF3;zG)Tp+oIa{UNXY*<0^ zo@VqRg>OcBj?tsRO{e`Q6p{|~Ash#AvJBc?O_yU@8gd}qZoD3KS~BD#i@QjJ)8sEC zw+)f1#>BTv{oy4fS8ZHck*tdrGcH=1UeS`AF^LAO=v%Ib?pt{A@GWQp-;xGBLCZ-9)$Elq|v9Nb7_=r3_)+<2-C z^HYL~9J+`KByBRA$y@l6D2jRwdPlN`#t_+{$|WOMMwg?lKC*Azb{;4MCh5n@C~zH) zKbQ>2973Ol2VI5*CBah72qas&{XDxUN(JN+UDP)tIZu30+R8$VS-dgp^ChaaPxkXG zr8qfwNkSj!UlB-CC`_g9oS>E_Df22IPeumhA@uh^mgM>j@DPd_NHsi1#wo$A^hUpI z2@LNuHgRcj@PMHE{cmuo608&;pn?|SYc~ zcO&p1i{zKk2}9Qk4+b?gfZ+0$eQZEC*oI8iQP z31CU*^Y@sUGC^I01J_ZTHGz>3_sz)ai-5;_teE1qIq%R0(a^RSJ0|{bSoLmH;3R z*86eGJVJ^oqI8I&z@=Q13}2KJ<~lj^p?C<%rm!OJ19`vtiHLJT$a2$TQyPtJ7j!HM z@q1ZDaUh99JFB{-r&NrsNHV7ONhR*XswApA8(5!UXUV_qeYE7|_4}yH?n1Io6gMJP zxnd^bbBE|XK*M7UTX=?fuwlxZ$h`3@bzH&MD^wH6w^TE?yA#O;idU zN(7yx_XG$~=k?ME4)~npXyO$vct}<=#R@jm1YJH}iW>!BO)Du zBxZIC7_xK{tJ^(s1k!5t-eB=2iYBgc+0Wm_r~d<2?nL5xiAVmGHIt07T1c^sk}Kvj z#J6h(^sN3<$mco8Yo%inu^bP8dV`()sBRwOw}ca6xrpbCX~|if(xqzq-6)H9a{`$) zAxU+@`8*|+5(BV3gb%q3x4=5o*N<=Lm+Er%oN|vZ6xSNZ$KI-tt%Vg(aU)5K7;`+7<>}x(0B@uKw`i{ zw>fQ+e;|co@^}|IF-asG&0S`ONF7!tnZUjcPEijVs6R@FMe-``e)YCN0npA-IF&ZM zZsbY#Zc}-{?ZpsEjIxXK3~}}gZh#}FofKP)QY)(=%ad!pb=?JA3j)iH2jewE+sZie z!X7UR75d0DVmXYekKbOS_l+8BJA!qL>0(kAXCOm-zLUC5JLqgM*t4>dZ>P_%tYC8Z zq-XWBs1D2*`Pl2tG;yrh;D-=Uhfh*vTU;Yd0170~4Q_ z8b?rizkxz5KBntiksf#D#O{Y^Yb@Y5IYcm{0^a*|KFHmV_Zn|eY%XFiB?i1|OnOM< z^q(3rq~T)HGQQtVH{@eHa*1c*%M=b5`@iUu7ObMK&+#QMA3Pg#jX%L9rnafqabW4V zbF^~IX*kC#Cn|GjokWt)jB}=p8x<;)1^`s3dRUWl!n?46YUYXdi)jyLTH>Hd@(g$! z32DaY0Qg~e;v?=0N(c6g6MHKUXs_AVeEp(P!; zh|}$&;zFLDv+|B5xVjL49m;_^_25vZDrm+@S48a`y>Eq+`4#9_R>nWRaaWckB#6=HLPrG%mG}8 zEZMPdV6%AXyZCW&t2IBkH0VXUHtuZ72R=lQ!o}}{fu*Sqw8Bk|7zLir&mNTGoBtbt zi%5AA_i@-C&L9R#)Vxv48fRG>kS9MMxb)bRN0scLKzo+r{bmi2tA9=_CIOO)KX{ diff --git a/contrib/site-packages/Pyro4/utils/flameserver.py b/contrib/site-packages/Pyro4/utils/flameserver.py deleted file mode 100644 index f5bc9226..00000000 --- a/contrib/site-packages/Pyro4/utils/flameserver.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -Pyro FLAME: Foreign Location Automatic Module Exposer. -Easy but potentially very dangerous way of exposing remote modules and builtins. -This is the commandline server. - -You can start this module as a script from the command line, to easily get a -flame server running: - - :command:`python -m Pyro4.utils.flameserver` - -Pyro - Python Remote Objects. Copyright by Irmen de Jong (irmen@razorvine.net). -""" - -import sys -import Pyro4.utils.flame -import Pyro4.core - - -def main(args, returnWithoutLooping=False): - from optparse import OptionParser - parser = OptionParser() - parser.add_option("-H", "--host", default="localhost", help="hostname to bind server on (default=localhost)") - parser.add_option("-p", "--port", type="int", default=0, help="port to bind server on") - parser.add_option("-u", "--unixsocket", help="Unix domain socket name to bind server on") - parser.add_option("-q", "--quiet", action="store_true", default=False, help="don't output anything") - parser.add_option("-k", "--key", help="the HMAC key to use") - options, args = parser.parse_args(args) - - if not options.quiet: - print("Starting Pyro Flame server.") - - hmac = (options.key or "").encode("utf-8") - if not hmac: - print("Warning: HMAC key not set. Anyone can connect to this server!") - Pyro4.config.HMAC_KEY = hmac or Pyro4.config.HMAC_KEY - if not options.quiet and Pyro4.config.HMAC_KEY: - print("HMAC_KEY set to: %s" % Pyro4.config.HMAC_KEY) - - Pyro4.config.SERIALIZERS_ACCEPTED = set(["pickle"]) # flame requires pickle serializer - - daemon = Pyro4.core.Daemon(host=options.host, port=options.port, unixsocket=options.unixsocket) - uri = Pyro4.utils.flame.start(daemon) - if not options.quiet: - print("server uri: %s" % uri) - print("server is running.") - - if returnWithoutLooping: - return daemon, uri # for unit testing - else: - daemon.requestLoop() - daemon.close() - return 0 - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) diff --git a/contrib/site-packages/Pyro4/utils/flameserver.pyo b/contrib/site-packages/Pyro4/utils/flameserver.pyo deleted file mode 100644 index 2bc426f342f2832641b47214607a3db9d35b1817..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2471 zcmc&${chV-6hC&_wDr<`bZa-*V0RnSGDIAK5TJ^oRV8JE?nCKdT7~dj-<#B_V|({r zZwqSw8V|r5AfAB7;13>v2jB@14*%CAjs55m1rH~>5F49o_j;>BY!q6h!(40@No><17i*O*#vl`MuaH%y zMDJx;7^TCY7n_OLsjMgqt8$yhSvC>Rm7WM0=R>87%7_>7L=*#|SSQVgLaQ-^#n?d^ z5$6&JX=c;hgu$cT)Chd-t`bQx9^*7ib0uI6uwf89DJqe~xiB`?7GjXCkwgqEVG^B| zRt$78K8_(6y^=jg+* zvT0^Qr&#;+G^n%Jfq)b3*YQ2r8L7mYP>A)Soal78YsJn)+|^^13#r7H1#;0&`SG@n zzc2K2xESWj-VB5EZ+z}2KI$`%E5NnRwfYscnzjMa*rNlVynUZu5gjyWzd<9PTLB$_ zV#_8;bi+Ko@_lc5hAs$5o*>zB5{pjaBqSP#5~oO>cHA@IdWYO4l4nVtBe~3a7AAna za$N5`$qSDA#%sM9+9z(eZ+$wLp#!4*85+%!`Hke8^u--CAv2m({!a2DjevNG#Sdm_ zA5wpkd}}H-OC@=Bo|||SP)$U^6t1P;KRN>~-=0c9bj4Y*i1#uLUF%PkTTW(?Mkh%M z2ZXqT%yrg8l9v4rt@%aYTNwe$!oMf$A;uPQ*S-ezjzP?dw?MW zHCpBhF>i#Nfs%_DzUwIHeOKr?vz*g7GyTg~a|yF|>{?{@b}4<(c6qJDq1$ zs;uSUpHe(_}AGF7>Y~>WaZ}A4};;cFHY7QR$S` z&C(GI+i<$Z!uENv(eFe>X-jUYK4@}zjHQgg<+O5|HsTE4upuWG#d>JmL0tsy0;J8k zsq&Z{ZcWv|TcCWt`t*x*39zN=Q?jlW4PKWS$ zjHQ|P*@gZIzHuu#D{kL9%I#c7x!sSkCerE|dckfMh=wCcGQ5}C9JIrCfv^WKQ|bJ1 ziUnOcX_UN92X~l@TD$S8vtsHxjL~|0E}eHSB;>pe4!N7rpt?}$x^e1mraL;;lkSGb z^t~wbp6S-_gm7@k>3)OBM&3>LgYe^SmhN=x7Ysj{)T&MyCwuWwu_j(OT^D}$*y|>Q zJkKZ36F0W`2siRhdkfwr@2a=ru4RA0oA)nxO>f@20N#?n?7!~m^e-_%E3&i*Oz`ZxHXhUZ}i5c{lN lruA|`U7jDM@qf)FJNaq-AH-+ebZ&_?{g#KnM(bM3`v*>ndO!dG diff --git a/contrib/site-packages/paramiko/__init__.py b/contrib/site-packages/paramiko/__init__.py deleted file mode 100644 index b1ef0e6f..00000000 --- a/contrib/site-packages/paramiko/__init__.py +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright (C) 2003-2011 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -I{Paramiko} (a combination of the esperanto words for "paranoid" and "friend") -is a module for python 2.5 or greater that implements the SSH2 protocol for -secure (encrypted and authenticated) connections to remote machines. Unlike -SSL (aka TLS), the SSH2 protocol does not require hierarchical certificates -signed by a powerful central authority. You may know SSH2 as the protocol that -replaced C{telnet} and C{rsh} for secure access to remote shells, but the -protocol also includes the ability to open arbitrary channels to remote -services across an encrypted tunnel. (This is how C{sftp} works, for example.) - -The high-level client API starts with creation of an L{SSHClient} object. -For more direct control, pass a socket (or socket-like object) to a -L{Transport}, and use L{start_server } or -L{start_client } to negoatite -with the remote host as either a server or client. As a client, you are -responsible for authenticating using a password or private key, and checking -the server's host key. I{(Key signature and verification is done by paramiko, -but you will need to provide private keys and check that the content of a -public key matches what you expected to see.)} As a server, you are -responsible for deciding which users, passwords, and keys to allow, and what -kind of channels to allow. - -Once you have finished, either side may request flow-controlled L{Channel}s to -the other side, which are python objects that act like sockets, but send and -receive data over the encrypted session. - -Paramiko is written entirely in python (no C or platform-dependent code) and is -released under the GNU Lesser General Public License (LGPL). - -Website: U{https://github.com/paramiko/paramiko/} - -Mailing list: U{paramiko@librelist.com} -""" - -import sys - -if sys.version_info < (2, 5): - raise RuntimeError('You need python 2.5+ for this module.') - - -__author__ = "Jeff Forcier " -__version__ = "1.12.0" -__license__ = "GNU Lesser General Public License (LGPL)" - - -from transport import SecurityOptions, Transport -from client import SSHClient, MissingHostKeyPolicy, AutoAddPolicy, RejectPolicy, WarningPolicy -from auth_handler import AuthHandler -from channel import Channel, ChannelFile -from ssh_exception import SSHException, PasswordRequiredException, \ - BadAuthenticationType, ChannelException, BadHostKeyException, \ - AuthenticationException, ProxyCommandFailure -from server import ServerInterface, SubsystemHandler, InteractiveQuery -from rsakey import RSAKey -from dsskey import DSSKey -from ecdsakey import ECDSAKey -from sftp import SFTPError, BaseSFTP -from sftp_client import SFTP, SFTPClient -from sftp_server import SFTPServer -from sftp_attr import SFTPAttributes -from sftp_handle import SFTPHandle -from sftp_si import SFTPServerInterface -from sftp_file import SFTPFile -from message import Message -from packet import Packetizer -from file import BufferedFile -from agent import Agent, AgentKey -from pkey import PKey -from hostkeys import HostKeys -from config import SSHConfig -from proxy import ProxyCommand - -# fix module names for epydoc -for c in locals().values(): - if issubclass(type(c), type) or type(c).__name__ == 'classobj': - # classobj for exceptions :/ - c.__module__ = __name__ -del c - -from common import AUTH_SUCCESSFUL, AUTH_PARTIALLY_SUCCESSFUL, AUTH_FAILED, \ - OPEN_SUCCEEDED, OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED, OPEN_FAILED_CONNECT_FAILED, \ - OPEN_FAILED_UNKNOWN_CHANNEL_TYPE, OPEN_FAILED_RESOURCE_SHORTAGE - -from sftp import SFTP_OK, SFTP_EOF, SFTP_NO_SUCH_FILE, SFTP_PERMISSION_DENIED, SFTP_FAILURE, \ - SFTP_BAD_MESSAGE, SFTP_NO_CONNECTION, SFTP_CONNECTION_LOST, SFTP_OP_UNSUPPORTED - -from common import io_sleep - -__all__ = [ 'Transport', - 'SSHClient', - 'MissingHostKeyPolicy', - 'AutoAddPolicy', - 'RejectPolicy', - 'WarningPolicy', - 'SecurityOptions', - 'SubsystemHandler', - 'Channel', - 'PKey', - 'RSAKey', - 'DSSKey', - 'Message', - 'SSHException', - 'AuthenticationException', - 'PasswordRequiredException', - 'BadAuthenticationType', - 'ChannelException', - 'BadHostKeyException', - 'ProxyCommand', - 'ProxyCommandFailure', - 'SFTP', - 'SFTPFile', - 'SFTPHandle', - 'SFTPClient', - 'SFTPServer', - 'SFTPError', - 'SFTPAttributes', - 'SFTPServerInterface', - 'ServerInterface', - 'BufferedFile', - 'Agent', - 'AgentKey', - 'HostKeys', - 'SSHConfig', - 'util', - 'io_sleep' ] diff --git a/contrib/site-packages/paramiko/_winapi.py b/contrib/site-packages/paramiko/_winapi.py deleted file mode 100644 index f141b005..00000000 --- a/contrib/site-packages/paramiko/_winapi.py +++ /dev/null @@ -1,269 +0,0 @@ -""" -Windows API functions implemented as ctypes functions and classes as found -in jaraco.windows (2.10). - -If you encounter issues with this module, please consider reporting the issues -in jaraco.windows and asking the author to port the fixes back here. -""" - -import ctypes -import ctypes.wintypes -import __builtin__ - -###################### -# jaraco.windows.error - -def format_system_message(errno): - """ - Call FormatMessage with a system error number to retrieve - the descriptive error message. - """ - # first some flags used by FormatMessageW - ALLOCATE_BUFFER = 0x100 - ARGUMENT_ARRAY = 0x2000 - FROM_HMODULE = 0x800 - FROM_STRING = 0x400 - FROM_SYSTEM = 0x1000 - IGNORE_INSERTS = 0x200 - - # Let FormatMessageW allocate the buffer (we'll free it below) - # Also, let it know we want a system error message. - flags = ALLOCATE_BUFFER | FROM_SYSTEM - source = None - message_id = errno - language_id = 0 - result_buffer = ctypes.wintypes.LPWSTR() - buffer_size = 0 - arguments = None - bytes = ctypes.windll.kernel32.FormatMessageW( - flags, - source, - message_id, - language_id, - ctypes.byref(result_buffer), - buffer_size, - arguments, - ) - # note the following will cause an infinite loop if GetLastError - # repeatedly returns an error that cannot be formatted, although - # this should not happen. - handle_nonzero_success(bytes) - message = result_buffer.value - ctypes.windll.kernel32.LocalFree(result_buffer) - return message - - -class WindowsError(__builtin__.WindowsError): - "more info about errors at http://msdn.microsoft.com/en-us/library/ms681381(VS.85).aspx" - - def __init__(self, value=None): - if value is None: - value = ctypes.windll.kernel32.GetLastError() - strerror = format_system_message(value) - super(WindowsError, self).__init__(value, strerror) - - @property - def message(self): - return self.strerror - - @property - def code(self): - return self.winerror - - def __str__(self): - return self.message - - def __repr__(self): - return '{self.__class__.__name__}({self.winerror})'.format(**vars()) - -def handle_nonzero_success(result): - if result == 0: - raise WindowsError() - - -##################### -# jaraco.windows.mmap - -CreateFileMapping = ctypes.windll.kernel32.CreateFileMappingW -CreateFileMapping.argtypes = [ - ctypes.wintypes.HANDLE, - ctypes.c_void_p, - ctypes.wintypes.DWORD, - ctypes.wintypes.DWORD, - ctypes.wintypes.DWORD, - ctypes.wintypes.LPWSTR, -] -CreateFileMapping.restype = ctypes.wintypes.HANDLE - -MapViewOfFile = ctypes.windll.kernel32.MapViewOfFile -MapViewOfFile.restype = ctypes.wintypes.HANDLE - -class MemoryMap(object): - """ - A memory map object which can have security attributes overrideden. - """ - def __init__(self, name, length, security_attributes=None): - self.name = name - self.length = length - self.security_attributes = security_attributes - self.pos = 0 - - def __enter__(self): - p_SA = ( - ctypes.byref(self.security_attributes) - if self.security_attributes else None - ) - INVALID_HANDLE_VALUE = -1 - PAGE_READWRITE = 0x4 - FILE_MAP_WRITE = 0x2 - filemap = ctypes.windll.kernel32.CreateFileMappingW( - INVALID_HANDLE_VALUE, p_SA, PAGE_READWRITE, 0, self.length, - unicode(self.name)) - handle_nonzero_success(filemap) - if filemap == INVALID_HANDLE_VALUE: - raise Exception("Failed to create file mapping") - self.filemap = filemap - self.view = MapViewOfFile(filemap, FILE_MAP_WRITE, 0, 0, 0) - return self - - def seek(self, pos): - self.pos = pos - - def write(self, msg): - ctypes.windll.msvcrt.memcpy(self.view + self.pos, msg, len(msg)) - self.pos += len(msg) - - def read(self, n): - """ - Read n bytes from mapped view. - """ - out = ctypes.create_string_buffer(n) - ctypes.windll.msvcrt.memcpy(out, self.view + self.pos, n) - self.pos += n - return out.raw - - def __exit__(self, exc_type, exc_val, tb): - ctypes.windll.kernel32.UnmapViewOfFile(self.view) - ctypes.windll.kernel32.CloseHandle(self.filemap) - -######################### -# jaraco.windows.security - -class TokenInformationClass: - TokenUser = 1 - -class TOKEN_USER(ctypes.Structure): - num = 1 - _fields_ = [ - ('SID', ctypes.c_void_p), - ('ATTRIBUTES', ctypes.wintypes.DWORD), - ] - - -class SECURITY_DESCRIPTOR(ctypes.Structure): - """ - typedef struct _SECURITY_DESCRIPTOR - { - UCHAR Revision; - UCHAR Sbz1; - SECURITY_DESCRIPTOR_CONTROL Control; - PSID Owner; - PSID Group; - PACL Sacl; - PACL Dacl; - } SECURITY_DESCRIPTOR; - """ - SECURITY_DESCRIPTOR_CONTROL = ctypes.wintypes.USHORT - REVISION = 1 - - _fields_ = [ - ('Revision', ctypes.c_ubyte), - ('Sbz1', ctypes.c_ubyte), - ('Control', SECURITY_DESCRIPTOR_CONTROL), - ('Owner', ctypes.c_void_p), - ('Group', ctypes.c_void_p), - ('Sacl', ctypes.c_void_p), - ('Dacl', ctypes.c_void_p), - ] - -class SECURITY_ATTRIBUTES(ctypes.Structure): - """ - typedef struct _SECURITY_ATTRIBUTES { - DWORD nLength; - LPVOID lpSecurityDescriptor; - BOOL bInheritHandle; - } SECURITY_ATTRIBUTES; - """ - _fields_ = [ - ('nLength', ctypes.wintypes.DWORD), - ('lpSecurityDescriptor', ctypes.c_void_p), - ('bInheritHandle', ctypes.wintypes.BOOL), - ] - - def __init__(self, *args, **kwargs): - super(SECURITY_ATTRIBUTES, self).__init__(*args, **kwargs) - self.nLength = ctypes.sizeof(SECURITY_ATTRIBUTES) - - def _get_descriptor(self): - return self._descriptor - def _set_descriptor(self, descriptor): - self._descriptor = descriptor - self.lpSecurityDescriptor = ctypes.addressof(descriptor) - descriptor = property(_get_descriptor, _set_descriptor) - -def GetTokenInformation(token, information_class): - """ - Given a token, get the token information for it. - """ - data_size = ctypes.wintypes.DWORD() - ctypes.windll.advapi32.GetTokenInformation(token, information_class.num, - 0, 0, ctypes.byref(data_size)) - data = ctypes.create_string_buffer(data_size.value) - handle_nonzero_success(ctypes.windll.advapi32.GetTokenInformation(token, - information_class.num, - ctypes.byref(data), ctypes.sizeof(data), - ctypes.byref(data_size))) - return ctypes.cast(data, ctypes.POINTER(TOKEN_USER)).contents - -class TokenAccess: - TOKEN_QUERY = 0x8 - -def OpenProcessToken(proc_handle, access): - result = ctypes.wintypes.HANDLE() - proc_handle = ctypes.wintypes.HANDLE(proc_handle) - handle_nonzero_success(ctypes.windll.advapi32.OpenProcessToken( - proc_handle, access, ctypes.byref(result))) - return result - -def get_current_user(): - """ - Return a TOKEN_USER for the owner of this process. - """ - process = OpenProcessToken( - ctypes.windll.kernel32.GetCurrentProcess(), - TokenAccess.TOKEN_QUERY, - ) - return GetTokenInformation(process, TOKEN_USER) - -def get_security_attributes_for_user(user=None): - """ - Return a SECURITY_ATTRIBUTES structure with the SID set to the - specified user (uses current user if none is specified). - """ - if user is None: - user = get_current_user() - - assert isinstance(user, TOKEN_USER), "user must be TOKEN_USER instance" - - SD = SECURITY_DESCRIPTOR() - SA = SECURITY_ATTRIBUTES() - # by attaching the actual security descriptor, it will be garbage- - # collected with the security attributes - SA.descriptor = SD - SA.bInheritHandle = 1 - - ctypes.windll.advapi32.InitializeSecurityDescriptor(ctypes.byref(SD), - SECURITY_DESCRIPTOR.REVISION) - ctypes.windll.advapi32.SetSecurityDescriptorOwner(ctypes.byref(SD), - user.SID, 0) - return SA diff --git a/contrib/site-packages/paramiko/agent.py b/contrib/site-packages/paramiko/agent.py deleted file mode 100644 index 23a5a2e4..00000000 --- a/contrib/site-packages/paramiko/agent.py +++ /dev/null @@ -1,380 +0,0 @@ -# Copyright (C) 2003-2007 John Rochester -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -SSH Agent interface for Unix clients. -""" - -import os -import socket -import struct -import sys -import threading -import time -import tempfile -import stat -from select import select - -from paramiko.ssh_exception import SSHException -from paramiko.message import Message -from paramiko.pkey import PKey -from paramiko.channel import Channel -from paramiko.common import io_sleep -from paramiko.util import retry_on_signal - -SSH2_AGENTC_REQUEST_IDENTITIES, SSH2_AGENT_IDENTITIES_ANSWER, \ - SSH2_AGENTC_SIGN_REQUEST, SSH2_AGENT_SIGN_RESPONSE = range(11, 15) - -class AgentSSH(object): - """ - Client interface for using private keys from an SSH agent running on the - local machine. If an SSH agent is running, this class can be used to - connect to it and retreive L{PKey} objects which can be used when - attempting to authenticate to remote SSH servers. - - Because the SSH agent protocol uses environment variables and unix-domain - sockets, this probably doesn't work on Windows. It does work on most - posix platforms though (Linux and MacOS X, for example). - """ - def __init__(self): - self._conn = None - self._keys = () - - def get_keys(self): - """ - Return the list of keys available through the SSH agent, if any. If - no SSH agent was running (or it couldn't be contacted), an empty list - will be returned. - - @return: a list of keys available on the SSH agent - @rtype: tuple of L{AgentKey} - """ - return self._keys - - def _connect(self, conn): - self._conn = conn - ptype, result = self._send_message(chr(SSH2_AGENTC_REQUEST_IDENTITIES)) - if ptype != SSH2_AGENT_IDENTITIES_ANSWER: - raise SSHException('could not get keys from ssh-agent') - keys = [] - for i in range(result.get_int()): - keys.append(AgentKey(self, result.get_string())) - result.get_string() - self._keys = tuple(keys) - - def _close(self): - #self._conn.close() - self._conn = None - self._keys = () - - def _send_message(self, msg): - msg = str(msg) - self._conn.send(struct.pack('>I', len(msg)) + msg) - l = self._read_all(4) - msg = Message(self._read_all(struct.unpack('>I', l)[0])) - return ord(msg.get_byte()), msg - - def _read_all(self, wanted): - result = self._conn.recv(wanted) - while len(result) < wanted: - if len(result) == 0: - raise SSHException('lost ssh-agent') - extra = self._conn.recv(wanted - len(result)) - if len(extra) == 0: - raise SSHException('lost ssh-agent') - result += extra - return result - -class AgentProxyThread(threading.Thread): - """ Class in charge of communication between two chan """ - def __init__(self, agent): - threading.Thread.__init__(self, target=self.run) - self._agent = agent - self._exit = False - - def run(self): - try: - (r,addr) = self.get_connection() - self.__inr = r - self.__addr = addr - self._agent.connect() - self._communicate() - except: - #XXX Not sure what to do here ... raise or pass ? - raise - - def _communicate(self): - import fcntl - oldflags = fcntl.fcntl(self.__inr, fcntl.F_GETFL) - fcntl.fcntl(self.__inr, fcntl.F_SETFL, oldflags | os.O_NONBLOCK) - while not self._exit: - events = select([self._agent._conn, self.__inr], [], [], 0.5) - for fd in events[0]: - if self._agent._conn == fd: - data = self._agent._conn.recv(512) - if len(data) != 0: - self.__inr.send(data) - else: - self._close() - break - elif self.__inr == fd: - data = self.__inr.recv(512) - if len(data) != 0: - self._agent._conn.send(data) - else: - self._close() - break - time.sleep(io_sleep) - - def _close(self): - self._exit = True - self.__inr.close() - self._agent._conn.close() - -class AgentLocalProxy(AgentProxyThread): - """ - Class to be used when wanting to ask a local SSH Agent being - asked from a remote fake agent (so use a unix socket for ex.) - """ - def __init__(self, agent): - AgentProxyThread.__init__(self, agent) - - def get_connection(self): - """ Return a pair of socket object and string address - May Block ! - """ - conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - try: - conn.bind(self._agent._get_filename()) - conn.listen(1) - (r,addr) = conn.accept() - return (r, addr) - except: - raise - return None - -class AgentRemoteProxy(AgentProxyThread): - """ - Class to be used when wanting to ask a remote SSH Agent - """ - def __init__(self, agent, chan): - AgentProxyThread.__init__(self, agent) - self.__chan = chan - - def get_connection(self): - """ - Class to be used when wanting to ask a local SSH Agent being - asked from a remote fake agent (so use a unix socket for ex.) - """ - return (self.__chan, None) - -class AgentClientProxy(object): - """ - Class proxying request as a client: - -> client ask for a request_forward_agent() - -> server creates a proxy and a fake SSH Agent - -> server ask for establishing a connection when needed, - calling the forward_agent_handler at client side. - -> the forward_agent_handler launch a thread for connecting - the remote fake agent and the local agent - -> Communication occurs ... - """ - def __init__(self, chanRemote): - self._conn = None - self.__chanR = chanRemote - self.thread = AgentRemoteProxy(self, chanRemote) - self.thread.start() - - def __del__(self): - self.close() - - def connect(self): - """ - Method automatically called by the run() method of the AgentProxyThread - """ - if ('SSH_AUTH_SOCK' in os.environ) and (sys.platform != 'win32'): - conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - try: - retry_on_signal(lambda: conn.connect(os.environ['SSH_AUTH_SOCK'])) - except: - # probably a dangling env var: the ssh agent is gone - return - elif sys.platform == 'win32': - import win_pageant - if win_pageant.can_talk_to_agent(): - conn = win_pageant.PageantConnection() - else: - return - else: - # no agent support - return - self._conn = conn - - def close(self): - """ - Close the current connection and terminate the agent - Should be called manually - """ - if hasattr(self, "thread"): - self.thread._exit = True - self.thread.join(1000) - if self._conn is not None: - self._conn.close() - -class AgentServerProxy(AgentSSH): - """ - @param t : transport used for the Forward for SSH Agent communication - - @raise SSHException: mostly if we lost the agent - """ - def __init__(self, t): - AgentSSH.__init__(self) - self.__t = t - self._dir = tempfile.mkdtemp('sshproxy') - os.chmod(self._dir, stat.S_IRWXU) - self._file = self._dir + '/sshproxy.ssh' - self.thread = AgentLocalProxy(self) - self.thread.start() - - def __del__(self): - self.close() - - def connect(self): - conn_sock = self.__t.open_forward_agent_channel() - if conn_sock is None: - raise SSHException('lost ssh-agent') - conn_sock.set_name('auth-agent') - self._connect(conn_sock) - - def close(self): - """ - Terminate the agent, clean the files, close connections - Should be called manually - """ - os.remove(self._file) - os.rmdir(self._dir) - self.thread._exit = True - self.thread.join(1000) - self._close() - - def get_env(self): - """ - Helper for the environnement under unix - - @return: the SSH_AUTH_SOCK Environnement variables - @rtype: dict - """ - env = {} - env['SSH_AUTH_SOCK'] = self._get_filename() - return env - - def _get_filename(self): - return self._file - -class AgentRequestHandler(object): - def __init__(self, chanClient): - self._conn = None - self.__chanC = chanClient - chanClient.request_forward_agent(self._forward_agent_handler) - self.__clientProxys = [] - - def _forward_agent_handler(self, chanRemote): - self.__clientProxys.append(AgentClientProxy(chanRemote)) - - def __del__(self): - self.close() - - def close(self): - for p in self.__clientProxys: - p.close() - -class Agent(AgentSSH): - """ - Client interface for using private keys from an SSH agent running on the - local machine. If an SSH agent is running, this class can be used to - connect to it and retreive L{PKey} objects which can be used when - attempting to authenticate to remote SSH servers. - - Because the SSH agent protocol uses environment variables and unix-domain - sockets, this probably doesn't work on Windows. It does work on most - posix platforms though (Linux and MacOS X, for example). - """ - - def __init__(self): - """ - Open a session with the local machine's SSH agent, if one is running. - If no agent is running, initialization will succeed, but L{get_keys} - will return an empty tuple. - - @raise SSHException: if an SSH agent is found, but speaks an - incompatible protocol - """ - AgentSSH.__init__(self) - - if ('SSH_AUTH_SOCK' in os.environ) and (sys.platform != 'win32'): - conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - try: - conn.connect(os.environ['SSH_AUTH_SOCK']) - except: - # probably a dangling env var: the ssh agent is gone - return - elif sys.platform == 'win32': - import win_pageant - if win_pageant.can_talk_to_agent(): - conn = win_pageant.PageantConnection() - else: - return - else: - # no agent support - return - self._connect(conn) - - def close(self): - """ - Close the SSH agent connection. - """ - self._close() - -class AgentKey(PKey): - """ - Private key held in a local SSH agent. This type of key can be used for - authenticating to a remote server (signing). Most other key operations - work as expected. - """ - - def __init__(self, agent, blob): - self.agent = agent - self.blob = blob - self.name = Message(blob).get_string() - - def __str__(self): - return self.blob - - def get_name(self): - return self.name - - def sign_ssh_data(self, rng, data): - msg = Message() - msg.add_byte(chr(SSH2_AGENTC_SIGN_REQUEST)) - msg.add_string(self.blob) - msg.add_string(data) - msg.add_int(0) - ptype, result = self.agent._send_message(msg) - if ptype != SSH2_AGENT_SIGN_RESPONSE: - raise SSHException('key cannot be used for signing') - return result.get_string() diff --git a/contrib/site-packages/paramiko/auth_handler.py b/contrib/site-packages/paramiko/auth_handler.py deleted file mode 100644 index acb7c8b8..00000000 --- a/contrib/site-packages/paramiko/auth_handler.py +++ /dev/null @@ -1,426 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -L{AuthHandler} -""" - -import threading -import weakref - -# this helps freezing utils -import encodings.utf_8 - -from paramiko.common import * -from paramiko import util -from paramiko.message import Message -from paramiko.ssh_exception import SSHException, AuthenticationException, \ - BadAuthenticationType, PartialAuthentication -from paramiko.server import InteractiveQuery - - -class AuthHandler (object): - """ - Internal class to handle the mechanics of authentication. - """ - - def __init__(self, transport): - self.transport = weakref.proxy(transport) - self.username = None - self.authenticated = False - self.auth_event = None - self.auth_method = '' - self.password = None - self.private_key = None - self.interactive_handler = None - self.submethods = None - # for server mode: - self.auth_username = None - self.auth_fail_count = 0 - - def is_authenticated(self): - return self.authenticated - - def get_username(self): - if self.transport.server_mode: - return self.auth_username - else: - return self.username - - def auth_none(self, username, event): - self.transport.lock.acquire() - try: - self.auth_event = event - self.auth_method = 'none' - self.username = username - self._request_auth() - finally: - self.transport.lock.release() - - def auth_publickey(self, username, key, event): - self.transport.lock.acquire() - try: - self.auth_event = event - self.auth_method = 'publickey' - self.username = username - self.private_key = key - self._request_auth() - finally: - self.transport.lock.release() - - def auth_password(self, username, password, event): - self.transport.lock.acquire() - try: - self.auth_event = event - self.auth_method = 'password' - self.username = username - self.password = password - self._request_auth() - finally: - self.transport.lock.release() - - def auth_interactive(self, username, handler, event, submethods=''): - """ - response_list = handler(title, instructions, prompt_list) - """ - self.transport.lock.acquire() - try: - self.auth_event = event - self.auth_method = 'keyboard-interactive' - self.username = username - self.interactive_handler = handler - self.submethods = submethods - self._request_auth() - finally: - self.transport.lock.release() - - def abort(self): - if self.auth_event is not None: - self.auth_event.set() - - - ### internals... - - - def _request_auth(self): - m = Message() - m.add_byte(chr(MSG_SERVICE_REQUEST)) - m.add_string('ssh-userauth') - self.transport._send_message(m) - - def _disconnect_service_not_available(self): - m = Message() - m.add_byte(chr(MSG_DISCONNECT)) - m.add_int(DISCONNECT_SERVICE_NOT_AVAILABLE) - m.add_string('Service not available') - m.add_string('en') - self.transport._send_message(m) - self.transport.close() - - def _disconnect_no_more_auth(self): - m = Message() - m.add_byte(chr(MSG_DISCONNECT)) - m.add_int(DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE) - m.add_string('No more auth methods available') - m.add_string('en') - self.transport._send_message(m) - self.transport.close() - - def _get_session_blob(self, key, service, username): - m = Message() - m.add_string(self.transport.session_id) - m.add_byte(chr(MSG_USERAUTH_REQUEST)) - m.add_string(username) - m.add_string(service) - m.add_string('publickey') - m.add_boolean(1) - m.add_string(key.get_name()) - m.add_string(str(key)) - return str(m) - - def wait_for_response(self, event): - while True: - event.wait(0.1) - if not self.transport.is_active(): - e = self.transport.get_exception() - if (e is None) or issubclass(e.__class__, EOFError): - e = AuthenticationException('Authentication failed.') - raise e - if event.isSet(): - break - if not self.is_authenticated(): - e = self.transport.get_exception() - if e is None: - e = AuthenticationException('Authentication failed.') - # this is horrible. python Exception isn't yet descended from - # object, so type(e) won't work. :( - if issubclass(e.__class__, PartialAuthentication): - return e.allowed_types - raise e - return [] - - def _parse_service_request(self, m): - service = m.get_string() - if self.transport.server_mode and (service == 'ssh-userauth'): - # accepted - m = Message() - m.add_byte(chr(MSG_SERVICE_ACCEPT)) - m.add_string(service) - self.transport._send_message(m) - return - # dunno this one - self._disconnect_service_not_available() - - def _parse_service_accept(self, m): - service = m.get_string() - if service == 'ssh-userauth': - self.transport._log(DEBUG, 'userauth is OK') - m = Message() - m.add_byte(chr(MSG_USERAUTH_REQUEST)) - m.add_string(self.username) - m.add_string('ssh-connection') - m.add_string(self.auth_method) - if self.auth_method == 'password': - m.add_boolean(False) - password = self.password - if isinstance(password, unicode): - password = password.encode('UTF-8') - m.add_string(password) - elif self.auth_method == 'publickey': - m.add_boolean(True) - m.add_string(self.private_key.get_name()) - m.add_string(str(self.private_key)) - blob = self._get_session_blob(self.private_key, 'ssh-connection', self.username) - sig = self.private_key.sign_ssh_data(self.transport.rng, blob) - m.add_string(str(sig)) - elif self.auth_method == 'keyboard-interactive': - m.add_string('') - m.add_string(self.submethods) - elif self.auth_method == 'none': - pass - else: - raise SSHException('Unknown auth method "%s"' % self.auth_method) - self.transport._send_message(m) - else: - self.transport._log(DEBUG, 'Service request "%s" accepted (?)' % service) - - def _send_auth_result(self, username, method, result): - # okay, send result - m = Message() - if result == AUTH_SUCCESSFUL: - self.transport._log(INFO, 'Auth granted (%s).' % method) - m.add_byte(chr(MSG_USERAUTH_SUCCESS)) - self.authenticated = True - else: - self.transport._log(INFO, 'Auth rejected (%s).' % method) - m.add_byte(chr(MSG_USERAUTH_FAILURE)) - m.add_string(self.transport.server_object.get_allowed_auths(username)) - if result == AUTH_PARTIALLY_SUCCESSFUL: - m.add_boolean(1) - else: - m.add_boolean(0) - self.auth_fail_count += 1 - self.transport._send_message(m) - if self.auth_fail_count >= 10: - self._disconnect_no_more_auth() - if result == AUTH_SUCCESSFUL: - self.transport._auth_trigger() - - def _interactive_query(self, q): - # make interactive query instead of response - m = Message() - m.add_byte(chr(MSG_USERAUTH_INFO_REQUEST)) - m.add_string(q.name) - m.add_string(q.instructions) - m.add_string('') - m.add_int(len(q.prompts)) - for p in q.prompts: - m.add_string(p[0]) - m.add_boolean(p[1]) - self.transport._send_message(m) - - def _parse_userauth_request(self, m): - if not self.transport.server_mode: - # er, uh... what? - m = Message() - m.add_byte(chr(MSG_USERAUTH_FAILURE)) - m.add_string('none') - m.add_boolean(0) - self.transport._send_message(m) - return - if self.authenticated: - # ignore - return - username = m.get_string() - service = m.get_string() - method = m.get_string() - self.transport._log(DEBUG, 'Auth request (type=%s) service=%s, username=%s' % (method, service, username)) - if service != 'ssh-connection': - self._disconnect_service_not_available() - return - if (self.auth_username is not None) and (self.auth_username != username): - self.transport._log(WARNING, 'Auth rejected because the client attempted to change username in mid-flight') - self._disconnect_no_more_auth() - return - self.auth_username = username - - if method == 'none': - result = self.transport.server_object.check_auth_none(username) - elif method == 'password': - changereq = m.get_boolean() - password = m.get_string() - try: - password = password.decode('UTF-8') - except UnicodeError: - # some clients/servers expect non-utf-8 passwords! - # in this case, just return the raw byte string. - pass - if changereq: - # always treated as failure, since we don't support changing passwords, but collect - # the list of valid auth types from the callback anyway - self.transport._log(DEBUG, 'Auth request to change passwords (rejected)') - newpassword = m.get_string() - try: - newpassword = newpassword.decode('UTF-8', 'replace') - except UnicodeError: - pass - result = AUTH_FAILED - else: - result = self.transport.server_object.check_auth_password(username, password) - elif method == 'publickey': - sig_attached = m.get_boolean() - keytype = m.get_string() - keyblob = m.get_string() - try: - key = self.transport._key_info[keytype](Message(keyblob)) - except SSHException, e: - self.transport._log(INFO, 'Auth rejected: public key: %s' % str(e)) - key = None - except: - self.transport._log(INFO, 'Auth rejected: unsupported or mangled public key') - key = None - if key is None: - self._disconnect_no_more_auth() - return - # first check if this key is okay... if not, we can skip the verify - result = self.transport.server_object.check_auth_publickey(username, key) - if result != AUTH_FAILED: - # key is okay, verify it - if not sig_attached: - # client wants to know if this key is acceptable, before it - # signs anything... send special "ok" message - m = Message() - m.add_byte(chr(MSG_USERAUTH_PK_OK)) - m.add_string(keytype) - m.add_string(keyblob) - self.transport._send_message(m) - return - sig = Message(m.get_string()) - blob = self._get_session_blob(key, service, username) - if not key.verify_ssh_sig(blob, sig): - self.transport._log(INFO, 'Auth rejected: invalid signature') - result = AUTH_FAILED - elif method == 'keyboard-interactive': - lang = m.get_string() - submethods = m.get_string() - result = self.transport.server_object.check_auth_interactive(username, submethods) - if isinstance(result, InteractiveQuery): - # make interactive query instead of response - self._interactive_query(result) - return - else: - result = self.transport.server_object.check_auth_none(username) - # okay, send result - self._send_auth_result(username, method, result) - - def _parse_userauth_success(self, m): - self.transport._log(INFO, 'Authentication (%s) successful!' % self.auth_method) - self.authenticated = True - self.transport._auth_trigger() - if self.auth_event != None: - self.auth_event.set() - - def _parse_userauth_failure(self, m): - authlist = m.get_list() - partial = m.get_boolean() - if partial: - self.transport._log(INFO, 'Authentication continues...') - self.transport._log(DEBUG, 'Methods: ' + str(authlist)) - self.transport.saved_exception = PartialAuthentication(authlist) - elif self.auth_method not in authlist: - self.transport._log(DEBUG, 'Authentication type (%s) not permitted.' % self.auth_method) - self.transport._log(DEBUG, 'Allowed methods: ' + str(authlist)) - self.transport.saved_exception = BadAuthenticationType('Bad authentication type', authlist) - else: - self.transport._log(INFO, 'Authentication (%s) failed.' % self.auth_method) - self.authenticated = False - self.username = None - if self.auth_event != None: - self.auth_event.set() - - def _parse_userauth_banner(self, m): - banner = m.get_string() - lang = m.get_string() - self.transport._log(INFO, 'Auth banner: ' + banner) - # who cares. - - def _parse_userauth_info_request(self, m): - if self.auth_method != 'keyboard-interactive': - raise SSHException('Illegal info request from server') - title = m.get_string() - instructions = m.get_string() - m.get_string() # lang - prompts = m.get_int() - prompt_list = [] - for i in range(prompts): - prompt_list.append((m.get_string(), m.get_boolean())) - response_list = self.interactive_handler(title, instructions, prompt_list) - - m = Message() - m.add_byte(chr(MSG_USERAUTH_INFO_RESPONSE)) - m.add_int(len(response_list)) - for r in response_list: - m.add_string(r) - self.transport._send_message(m) - - def _parse_userauth_info_response(self, m): - if not self.transport.server_mode: - raise SSHException('Illegal info response from server') - n = m.get_int() - responses = [] - for i in range(n): - responses.append(m.get_string()) - result = self.transport.server_object.check_auth_interactive_response(responses) - if isinstance(type(result), InteractiveQuery): - # make interactive query instead of response - self._interactive_query(result) - return - self._send_auth_result(self.auth_username, 'keyboard-interactive', result) - - - _handler_table = { - MSG_SERVICE_REQUEST: _parse_service_request, - MSG_SERVICE_ACCEPT: _parse_service_accept, - MSG_USERAUTH_REQUEST: _parse_userauth_request, - MSG_USERAUTH_SUCCESS: _parse_userauth_success, - MSG_USERAUTH_FAILURE: _parse_userauth_failure, - MSG_USERAUTH_BANNER: _parse_userauth_banner, - MSG_USERAUTH_INFO_REQUEST: _parse_userauth_info_request, - MSG_USERAUTH_INFO_RESPONSE: _parse_userauth_info_response, - } - diff --git a/contrib/site-packages/paramiko/ber.py b/contrib/site-packages/paramiko/ber.py deleted file mode 100644 index 3941581c..00000000 --- a/contrib/site-packages/paramiko/ber.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - - -import util - - -class BERException (Exception): - pass - - -class BER(object): - """ - Robey's tiny little attempt at a BER decoder. - """ - - def __init__(self, content=''): - self.content = content - self.idx = 0 - - def __str__(self): - return self.content - - def __repr__(self): - return 'BER(\'' + repr(self.content) + '\')' - - def decode(self): - return self.decode_next() - - def decode_next(self): - if self.idx >= len(self.content): - return None - ident = ord(self.content[self.idx]) - self.idx += 1 - if (ident & 31) == 31: - # identifier > 30 - ident = 0 - while self.idx < len(self.content): - t = ord(self.content[self.idx]) - self.idx += 1 - ident = (ident << 7) | (t & 0x7f) - if not (t & 0x80): - break - if self.idx >= len(self.content): - return None - # now fetch length - size = ord(self.content[self.idx]) - self.idx += 1 - if size & 0x80: - # more complimicated... - # FIXME: theoretically should handle indefinite-length (0x80) - t = size & 0x7f - if self.idx + t > len(self.content): - return None - size = util.inflate_long(self.content[self.idx : self.idx + t], True) - self.idx += t - if self.idx + size > len(self.content): - # can't fit - return None - data = self.content[self.idx : self.idx + size] - self.idx += size - # now switch on id - if ident == 0x30: - # sequence - return self.decode_sequence(data) - elif ident == 2: - # int - return util.inflate_long(data) - else: - # 1: boolean (00 false, otherwise true) - raise BERException('Unknown ber encoding type %d (robey is lazy)' % ident) - - def decode_sequence(data): - out = [] - b = BER(data) - while True: - x = b.decode_next() - if x is None: - break - out.append(x) - return out - decode_sequence = staticmethod(decode_sequence) - - def encode_tlv(self, ident, val): - # no need to support ident > 31 here - self.content += chr(ident) - if len(val) > 0x7f: - lenstr = util.deflate_long(len(val)) - self.content += chr(0x80 + len(lenstr)) + lenstr - else: - self.content += chr(len(val)) - self.content += val - - def encode(self, x): - if type(x) is bool: - if x: - self.encode_tlv(1, '\xff') - else: - self.encode_tlv(1, '\x00') - elif (type(x) is int) or (type(x) is long): - self.encode_tlv(2, util.deflate_long(x)) - elif type(x) is str: - self.encode_tlv(4, x) - elif (type(x) is list) or (type(x) is tuple): - self.encode_tlv(0x30, self.encode_sequence(x)) - else: - raise BERException('Unknown type for encoding: %s' % repr(type(x))) - - def encode_sequence(data): - b = BER() - for item in data: - b.encode(item) - return str(b) - encode_sequence = staticmethod(encode_sequence) diff --git a/contrib/site-packages/paramiko/buffered_pipe.py b/contrib/site-packages/paramiko/buffered_pipe.py deleted file mode 100644 index 4ef5cf74..00000000 --- a/contrib/site-packages/paramiko/buffered_pipe.py +++ /dev/null @@ -1,200 +0,0 @@ -# Copyright (C) 2006-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -Attempt to generalize the "feeder" part of a Channel: an object which can be -read from and closed, but is reading from a buffer fed by another thread. The -read operations are blocking and can have a timeout set. -""" - -import array -import threading -import time - - -class PipeTimeout (IOError): - """ - Indicates that a timeout was reached on a read from a L{BufferedPipe}. - """ - pass - - -class BufferedPipe (object): - """ - A buffer that obeys normal read (with timeout) & close semantics for a - file or socket, but is fed data from another thread. This is used by - L{Channel}. - """ - - def __init__(self): - self._lock = threading.Lock() - self._cv = threading.Condition(self._lock) - self._event = None - self._buffer = array.array('B') - self._closed = False - - def set_event(self, event): - """ - Set an event on this buffer. When data is ready to be read (or the - buffer has been closed), the event will be set. When no data is - ready, the event will be cleared. - - @param event: the event to set/clear - @type event: Event - """ - self._event = event - if len(self._buffer) > 0: - event.set() - else: - event.clear() - - def feed(self, data): - """ - Feed new data into this pipe. This method is assumed to be called - from a separate thread, so synchronization is done. - - @param data: the data to add - @type data: str - """ - self._lock.acquire() - try: - if self._event is not None: - self._event.set() - self._buffer.fromstring(data) - self._cv.notifyAll() - finally: - self._lock.release() - - def read_ready(self): - """ - Returns true if data is buffered and ready to be read from this - feeder. A C{False} result does not mean that the feeder has closed; - it means you may need to wait before more data arrives. - - @return: C{True} if a L{read} call would immediately return at least - one byte; C{False} otherwise. - @rtype: bool - """ - self._lock.acquire() - try: - if len(self._buffer) == 0: - return False - return True - finally: - self._lock.release() - - def read(self, nbytes, timeout=None): - """ - Read data from the pipe. The return value is a string representing - the data received. The maximum amount of data to be received at once - is specified by C{nbytes}. If a string of length zero is returned, - the pipe has been closed. - - The optional C{timeout} argument can be a nonnegative float expressing - seconds, or C{None} for no timeout. If a float is given, a - C{PipeTimeout} will be raised if the timeout period value has - elapsed before any data arrives. - - @param nbytes: maximum number of bytes to read - @type nbytes: int - @param timeout: maximum seconds to wait (or C{None}, the default, to - wait forever) - @type timeout: float - @return: data - @rtype: str - - @raise PipeTimeout: if a timeout was specified and no data was ready - before that timeout - """ - out = '' - self._lock.acquire() - try: - if len(self._buffer) == 0: - if self._closed: - return out - # should we block? - if timeout == 0.0: - raise PipeTimeout() - # loop here in case we get woken up but a different thread has - # grabbed everything in the buffer. - while (len(self._buffer) == 0) and not self._closed: - then = time.time() - self._cv.wait(timeout) - if timeout is not None: - timeout -= time.time() - then - if timeout <= 0.0: - raise PipeTimeout() - - # something's in the buffer and we have the lock! - if len(self._buffer) <= nbytes: - out = self._buffer.tostring() - del self._buffer[:] - if (self._event is not None) and not self._closed: - self._event.clear() - else: - out = self._buffer[:nbytes].tostring() - del self._buffer[:nbytes] - finally: - self._lock.release() - - return out - - def empty(self): - """ - Clear out the buffer and return all data that was in it. - - @return: any data that was in the buffer prior to clearing it out - @rtype: str - """ - self._lock.acquire() - try: - out = self._buffer.tostring() - del self._buffer[:] - if (self._event is not None) and not self._closed: - self._event.clear() - return out - finally: - self._lock.release() - - def close(self): - """ - Close this pipe object. Future calls to L{read} after the buffer - has been emptied will return immediately with an empty string. - """ - self._lock.acquire() - try: - self._closed = True - self._cv.notifyAll() - if self._event is not None: - self._event.set() - finally: - self._lock.release() - - def __len__(self): - """ - Return the number of bytes buffered. - - @return: number of bytes bufferes - @rtype: int - """ - self._lock.acquire() - try: - return len(self._buffer) - finally: - self._lock.release() - diff --git a/contrib/site-packages/paramiko/channel.py b/contrib/site-packages/paramiko/channel.py deleted file mode 100644 index d1e6333c..00000000 --- a/contrib/site-packages/paramiko/channel.py +++ /dev/null @@ -1,1279 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -Abstraction for an SSH2 channel. -""" - -import binascii -import sys -import time -import threading -import socket -import os - -from paramiko.common import * -from paramiko import util -from paramiko.message import Message -from paramiko.ssh_exception import SSHException -from paramiko.file import BufferedFile -from paramiko.buffered_pipe import BufferedPipe, PipeTimeout -from paramiko import pipe - - -# lower bound on the max packet size we'll accept from the remote host -MIN_PACKET_SIZE = 1024 - - -class Channel (object): - """ - A secure tunnel across an SSH L{Transport}. A Channel is meant to behave - like a socket, and has an API that should be indistinguishable from the - python socket API. - - Because SSH2 has a windowing kind of flow control, if you stop reading data - from a Channel and its buffer fills up, the server will be unable to send - you any more data until you read some of it. (This won't affect other - channels on the same transport -- all channels on a single transport are - flow-controlled independently.) Similarly, if the server isn't reading - data you send, calls to L{send} may block, unless you set a timeout. This - is exactly like a normal network socket, so it shouldn't be too surprising. - """ - - def __init__(self, chanid): - """ - Create a new channel. The channel is not associated with any - particular session or L{Transport} until the Transport attaches it. - Normally you would only call this method from the constructor of a - subclass of L{Channel}. - - @param chanid: the ID of this channel, as passed by an existing - L{Transport}. - @type chanid: int - """ - self.chanid = chanid - self.remote_chanid = 0 - self.transport = None - self.active = False - self.eof_received = 0 - self.eof_sent = 0 - self.in_buffer = BufferedPipe() - self.in_stderr_buffer = BufferedPipe() - self.timeout = None - self.closed = False - self.ultra_debug = False - self.lock = threading.Lock() - self.out_buffer_cv = threading.Condition(self.lock) - self.in_window_size = 0 - self.out_window_size = 0 - self.in_max_packet_size = 0 - self.out_max_packet_size = 0 - self.in_window_threshold = 0 - self.in_window_sofar = 0 - self.status_event = threading.Event() - self._name = str(chanid) - self.logger = util.get_logger('paramiko.transport') - self._pipe = None - self.event = threading.Event() - self.event_ready = False - self.combine_stderr = False - self.exit_status = -1 - self.origin_addr = None - - def __del__(self): - try: - self.close() - except: - pass - - def __repr__(self): - """ - Return a string representation of this object, for debugging. - - @rtype: str - """ - out = ' 0: - out += ' in-buffer=%d' % (len(self.in_buffer),) - out += ' -> ' + repr(self.transport) - out += '>' - return out - - def get_pty(self, term='vt100', width=80, height=24, width_pixels=0, - height_pixels=0): - """ - Request a pseudo-terminal from the server. This is usually used right - after creating a client channel, to ask the server to provide some - basic terminal semantics for a shell invoked with L{invoke_shell}. - It isn't necessary (or desirable) to call this method if you're going - to exectue a single command with L{exec_command}. - - @param term: the terminal type to emulate (for example, C{'vt100'}) - @type term: str - @param width: width (in characters) of the terminal screen - @type width: int - @param height: height (in characters) of the terminal screen - @type height: int - @param width_pixels: width (in pixels) of the terminal screen - @type width_pixels: int - @param height_pixels: height (in pixels) of the terminal screen - @type height_pixels: int - - @raise SSHException: if the request was rejected or the channel was - closed - """ - if self.closed or self.eof_received or self.eof_sent or not self.active: - raise SSHException('Channel is not open') - m = Message() - m.add_byte(chr(MSG_CHANNEL_REQUEST)) - m.add_int(self.remote_chanid) - m.add_string('pty-req') - m.add_boolean(True) - m.add_string(term) - m.add_int(width) - m.add_int(height) - m.add_int(width_pixels) - m.add_int(height_pixels) - m.add_string('') - self._event_pending() - self.transport._send_user_message(m) - self._wait_for_event() - - def invoke_shell(self): - """ - Request an interactive shell session on this channel. If the server - allows it, the channel will then be directly connected to the stdin, - stdout, and stderr of the shell. - - Normally you would call L{get_pty} before this, in which case the - shell will operate through the pty, and the channel will be connected - to the stdin and stdout of the pty. - - When the shell exits, the channel will be closed and can't be reused. - You must open a new channel if you wish to open another shell. - - @raise SSHException: if the request was rejected or the channel was - closed - """ - if self.closed or self.eof_received or self.eof_sent or not self.active: - raise SSHException('Channel is not open') - m = Message() - m.add_byte(chr(MSG_CHANNEL_REQUEST)) - m.add_int(self.remote_chanid) - m.add_string('shell') - m.add_boolean(1) - self._event_pending() - self.transport._send_user_message(m) - self._wait_for_event() - - def exec_command(self, command): - """ - Execute a command on the server. If the server allows it, the channel - will then be directly connected to the stdin, stdout, and stderr of - the command being executed. - - When the command finishes executing, the channel will be closed and - can't be reused. You must open a new channel if you wish to execute - another command. - - @param command: a shell command to execute. - @type command: str - - @raise SSHException: if the request was rejected or the channel was - closed - """ - if self.closed or self.eof_received or self.eof_sent or not self.active: - raise SSHException('Channel is not open') - m = Message() - m.add_byte(chr(MSG_CHANNEL_REQUEST)) - m.add_int(self.remote_chanid) - m.add_string('exec') - m.add_boolean(True) - m.add_string(command) - self._event_pending() - self.transport._send_user_message(m) - self._wait_for_event() - - def invoke_subsystem(self, subsystem): - """ - Request a subsystem on the server (for example, C{sftp}). If the - server allows it, the channel will then be directly connected to the - requested subsystem. - - When the subsystem finishes, the channel will be closed and can't be - reused. - - @param subsystem: name of the subsystem being requested. - @type subsystem: str - - @raise SSHException: if the request was rejected or the channel was - closed - """ - if self.closed or self.eof_received or self.eof_sent or not self.active: - raise SSHException('Channel is not open') - m = Message() - m.add_byte(chr(MSG_CHANNEL_REQUEST)) - m.add_int(self.remote_chanid) - m.add_string('subsystem') - m.add_boolean(True) - m.add_string(subsystem) - self._event_pending() - self.transport._send_user_message(m) - self._wait_for_event() - - def resize_pty(self, width=80, height=24, width_pixels=0, height_pixels=0): - """ - Resize the pseudo-terminal. This can be used to change the width and - height of the terminal emulation created in a previous L{get_pty} call. - - @param width: new width (in characters) of the terminal screen - @type width: int - @param height: new height (in characters) of the terminal screen - @type height: int - @param width_pixels: new width (in pixels) of the terminal screen - @type width_pixels: int - @param height_pixels: new height (in pixels) of the terminal screen - @type height_pixels: int - - @raise SSHException: if the request was rejected or the channel was - closed - """ - if self.closed or self.eof_received or self.eof_sent or not self.active: - raise SSHException('Channel is not open') - m = Message() - m.add_byte(chr(MSG_CHANNEL_REQUEST)) - m.add_int(self.remote_chanid) - m.add_string('window-change') - m.add_boolean(False) - m.add_int(width) - m.add_int(height) - m.add_int(width_pixels) - m.add_int(height_pixels) - self.transport._send_user_message(m) - - def exit_status_ready(self): - """ - Return true if the remote process has exited and returned an exit - status. You may use this to poll the process status if you don't - want to block in L{recv_exit_status}. Note that the server may not - return an exit status in some cases (like bad servers). - - @return: True if L{recv_exit_status} will return immediately - @rtype: bool - @since: 1.7.3 - """ - return self.closed or self.status_event.isSet() - - def recv_exit_status(self): - """ - Return the exit status from the process on the server. This is - mostly useful for retrieving the reults of an L{exec_command}. - If the command hasn't finished yet, this method will wait until - it does, or until the channel is closed. If no exit status is - provided by the server, -1 is returned. - - @return: the exit code of the process on the server. - @rtype: int - - @since: 1.2 - """ - self.status_event.wait() - assert self.status_event.isSet() - return self.exit_status - - def send_exit_status(self, status): - """ - Send the exit status of an executed command to the client. (This - really only makes sense in server mode.) Many clients expect to - get some sort of status code back from an executed command after - it completes. - - @param status: the exit code of the process - @type status: int - - @since: 1.2 - """ - # in many cases, the channel will not still be open here. - # that's fine. - m = Message() - m.add_byte(chr(MSG_CHANNEL_REQUEST)) - m.add_int(self.remote_chanid) - m.add_string('exit-status') - m.add_boolean(False) - m.add_int(status) - self.transport._send_user_message(m) - - def request_x11(self, screen_number=0, auth_protocol=None, auth_cookie=None, - single_connection=False, handler=None): - """ - Request an x11 session on this channel. If the server allows it, - further x11 requests can be made from the server to the client, - when an x11 application is run in a shell session. - - From RFC4254:: - - It is RECOMMENDED that the 'x11 authentication cookie' that is - sent be a fake, random cookie, and that the cookie be checked and - replaced by the real cookie when a connection request is received. - - If you omit the auth_cookie, a new secure random 128-bit value will be - generated, used, and returned. You will need to use this value to - verify incoming x11 requests and replace them with the actual local - x11 cookie (which requires some knoweldge of the x11 protocol). - - If a handler is passed in, the handler is called from another thread - whenever a new x11 connection arrives. The default handler queues up - incoming x11 connections, which may be retrieved using - L{Transport.accept}. The handler's calling signature is:: - - handler(channel: Channel, (address: str, port: int)) - - @param screen_number: the x11 screen number (0, 10, etc) - @type screen_number: int - @param auth_protocol: the name of the X11 authentication method used; - if none is given, C{"MIT-MAGIC-COOKIE-1"} is used - @type auth_protocol: str - @param auth_cookie: hexadecimal string containing the x11 auth cookie; - if none is given, a secure random 128-bit value is generated - @type auth_cookie: str - @param single_connection: if True, only a single x11 connection will be - forwarded (by default, any number of x11 connections can arrive - over this session) - @type single_connection: bool - @param handler: an optional handler to use for incoming X11 connections - @type handler: function - @return: the auth_cookie used - """ - if self.closed or self.eof_received or self.eof_sent or not self.active: - raise SSHException('Channel is not open') - if auth_protocol is None: - auth_protocol = 'MIT-MAGIC-COOKIE-1' - if auth_cookie is None: - auth_cookie = binascii.hexlify(self.transport.rng.read(16)) - - m = Message() - m.add_byte(chr(MSG_CHANNEL_REQUEST)) - m.add_int(self.remote_chanid) - m.add_string('x11-req') - m.add_boolean(True) - m.add_boolean(single_connection) - m.add_string(auth_protocol) - m.add_string(auth_cookie) - m.add_int(screen_number) - self._event_pending() - self.transport._send_user_message(m) - self._wait_for_event() - self.transport._set_x11_handler(handler) - return auth_cookie - - def request_forward_agent(self, handler): - """ - Request for a forward SSH Agent on this channel. - This is only valid for an ssh-agent from openssh !!! - - @param handler: a required handler to use for incoming SSH Agent connections - @type handler: function - - @return: if we are ok or not (at that time we always return ok) - @rtype: boolean - - @raise: SSHException in case of channel problem. - """ - if self.closed or self.eof_received or self.eof_sent or not self.active: - raise SSHException('Channel is not open') - - m = Message() - m.add_byte(chr(MSG_CHANNEL_REQUEST)) - m.add_int(self.remote_chanid) - m.add_string('auth-agent-req@openssh.com') - m.add_boolean(False) - self.transport._send_user_message(m) - self.transport._set_forward_agent_handler(handler) - return True - - def get_transport(self): - """ - Return the L{Transport} associated with this channel. - - @return: the L{Transport} that was used to create this channel. - @rtype: L{Transport} - """ - return self.transport - - def set_name(self, name): - """ - Set a name for this channel. Currently it's only used to set the name - of the channel in logfile entries. The name can be fetched with the - L{get_name} method. - - @param name: new channel name - @type name: str - """ - self._name = name - - def get_name(self): - """ - Get the name of this channel that was previously set by L{set_name}. - - @return: the name of this channel. - @rtype: str - """ - return self._name - - def get_id(self): - """ - Return the ID # for this channel. The channel ID is unique across - a L{Transport} and usually a small number. It's also the number - passed to L{ServerInterface.check_channel_request} when determining - whether to accept a channel request in server mode. - - @return: the ID of this channel. - @rtype: int - """ - return self.chanid - - def set_combine_stderr(self, combine): - """ - Set whether stderr should be combined into stdout on this channel. - The default is C{False}, but in some cases it may be convenient to - have both streams combined. - - If this is C{False}, and L{exec_command} is called (or C{invoke_shell} - with no pty), output to stderr will not show up through the L{recv} - and L{recv_ready} calls. You will have to use L{recv_stderr} and - L{recv_stderr_ready} to get stderr output. - - If this is C{True}, data will never show up via L{recv_stderr} or - L{recv_stderr_ready}. - - @param combine: C{True} if stderr output should be combined into - stdout on this channel. - @type combine: bool - @return: previous setting. - @rtype: bool - - @since: 1.1 - """ - data = '' - self.lock.acquire() - try: - old = self.combine_stderr - self.combine_stderr = combine - if combine and not old: - # copy old stderr buffer into primary buffer - data = self.in_stderr_buffer.empty() - finally: - self.lock.release() - if len(data) > 0: - self._feed(data) - return old - - - ### socket API - - - def settimeout(self, timeout): - """ - Set a timeout on blocking read/write operations. The C{timeout} - argument can be a nonnegative float expressing seconds, or C{None}. If - a float is given, subsequent channel read/write operations will raise - a timeout exception if the timeout period value has elapsed before the - operation has completed. Setting a timeout of C{None} disables - timeouts on socket operations. - - C{chan.settimeout(0.0)} is equivalent to C{chan.setblocking(0)}; - C{chan.settimeout(None)} is equivalent to C{chan.setblocking(1)}. - - @param timeout: seconds to wait for a pending read/write operation - before raising C{socket.timeout}, or C{None} for no timeout. - @type timeout: float - """ - self.timeout = timeout - - def gettimeout(self): - """ - Returns the timeout in seconds (as a float) associated with socket - operations, or C{None} if no timeout is set. This reflects the last - call to L{setblocking} or L{settimeout}. - - @return: timeout in seconds, or C{None}. - @rtype: float - """ - return self.timeout - - def setblocking(self, blocking): - """ - Set blocking or non-blocking mode of the channel: if C{blocking} is 0, - the channel is set to non-blocking mode; otherwise it's set to blocking - mode. Initially all channels are in blocking mode. - - In non-blocking mode, if a L{recv} call doesn't find any data, or if a - L{send} call can't immediately dispose of the data, an error exception - is raised. In blocking mode, the calls block until they can proceed. An - EOF condition is considered "immediate data" for L{recv}, so if the - channel is closed in the read direction, it will never block. - - C{chan.setblocking(0)} is equivalent to C{chan.settimeout(0)}; - C{chan.setblocking(1)} is equivalent to C{chan.settimeout(None)}. - - @param blocking: 0 to set non-blocking mode; non-0 to set blocking - mode. - @type blocking: int - """ - if blocking: - self.settimeout(None) - else: - self.settimeout(0.0) - - def getpeername(self): - """ - Return the address of the remote side of this Channel, if possible. - This is just a wrapper around C{'getpeername'} on the Transport, used - to provide enough of a socket-like interface to allow asyncore to work. - (asyncore likes to call C{'getpeername'}.) - - @return: the address if the remote host, if known - @rtype: tuple(str, int) - """ - return self.transport.getpeername() - - def close(self): - """ - Close the channel. All future read/write operations on the channel - will fail. The remote end will receive no more data (after queued data - is flushed). Channels are automatically closed when their L{Transport} - is closed or when they are garbage collected. - """ - self.lock.acquire() - try: - # only close the pipe when the user explicitly closes the channel. - # otherwise they will get unpleasant surprises. (and do it before - # checking self.closed, since the remote host may have already - # closed the connection.) - if self._pipe is not None: - self._pipe.close() - self._pipe = None - - if not self.active or self.closed: - return - msgs = self._close_internal() - finally: - self.lock.release() - for m in msgs: - if m is not None: - self.transport._send_user_message(m) - - def recv_ready(self): - """ - Returns true if data is buffered and ready to be read from this - channel. A C{False} result does not mean that the channel has closed; - it means you may need to wait before more data arrives. - - @return: C{True} if a L{recv} call on this channel would immediately - return at least one byte; C{False} otherwise. - @rtype: boolean - """ - return self.in_buffer.read_ready() - - def recv(self, nbytes): - """ - Receive data from the channel. The return value is a string - representing the data received. The maximum amount of data to be - received at once is specified by C{nbytes}. If a string of length zero - is returned, the channel stream has closed. - - @param nbytes: maximum number of bytes to read. - @type nbytes: int - @return: data. - @rtype: str - - @raise socket.timeout: if no data is ready before the timeout set by - L{settimeout}. - """ - try: - out = self.in_buffer.read(nbytes, self.timeout) - except PipeTimeout, e: - raise socket.timeout() - - ack = self._check_add_window(len(out)) - # no need to hold the channel lock when sending this - if ack > 0: - m = Message() - m.add_byte(chr(MSG_CHANNEL_WINDOW_ADJUST)) - m.add_int(self.remote_chanid) - m.add_int(ack) - self.transport._send_user_message(m) - - return out - - def recv_stderr_ready(self): - """ - Returns true if data is buffered and ready to be read from this - channel's stderr stream. Only channels using L{exec_command} or - L{invoke_shell} without a pty will ever have data on the stderr - stream. - - @return: C{True} if a L{recv_stderr} call on this channel would - immediately return at least one byte; C{False} otherwise. - @rtype: boolean - - @since: 1.1 - """ - return self.in_stderr_buffer.read_ready() - - def recv_stderr(self, nbytes): - """ - Receive data from the channel's stderr stream. Only channels using - L{exec_command} or L{invoke_shell} without a pty will ever have data - on the stderr stream. The return value is a string representing the - data received. The maximum amount of data to be received at once is - specified by C{nbytes}. If a string of length zero is returned, the - channel stream has closed. - - @param nbytes: maximum number of bytes to read. - @type nbytes: int - @return: data. - @rtype: str - - @raise socket.timeout: if no data is ready before the timeout set by - L{settimeout}. - - @since: 1.1 - """ - try: - out = self.in_stderr_buffer.read(nbytes, self.timeout) - except PipeTimeout, e: - raise socket.timeout() - - ack = self._check_add_window(len(out)) - # no need to hold the channel lock when sending this - if ack > 0: - m = Message() - m.add_byte(chr(MSG_CHANNEL_WINDOW_ADJUST)) - m.add_int(self.remote_chanid) - m.add_int(ack) - self.transport._send_user_message(m) - - return out - - def send_ready(self): - """ - Returns true if data can be written to this channel without blocking. - This means the channel is either closed (so any write attempt would - return immediately) or there is at least one byte of space in the - outbound buffer. If there is at least one byte of space in the - outbound buffer, a L{send} call will succeed immediately and return - the number of bytes actually written. - - @return: C{True} if a L{send} call on this channel would immediately - succeed or fail - @rtype: boolean - """ - self.lock.acquire() - try: - if self.closed or self.eof_sent: - return True - return self.out_window_size > 0 - finally: - self.lock.release() - - def send(self, s): - """ - Send data to the channel. Returns the number of bytes sent, or 0 if - the channel stream is closed. Applications are responsible for - checking that all data has been sent: if only some of the data was - transmitted, the application needs to attempt delivery of the remaining - data. - - @param s: data to send - @type s: str - @return: number of bytes actually sent - @rtype: int - - @raise socket.timeout: if no data could be sent before the timeout set - by L{settimeout}. - """ - size = len(s) - self.lock.acquire() - try: - size = self._wait_for_send_window(size) - if size == 0: - # eof or similar - return 0 - m = Message() - m.add_byte(chr(MSG_CHANNEL_DATA)) - m.add_int(self.remote_chanid) - m.add_string(s[:size]) - finally: - self.lock.release() - # Note: We release self.lock before calling _send_user_message. - # Otherwise, we can deadlock during re-keying. - self.transport._send_user_message(m) - return size - - def send_stderr(self, s): - """ - Send data to the channel on the "stderr" stream. This is normally - only used by servers to send output from shell commands -- clients - won't use this. Returns the number of bytes sent, or 0 if the channel - stream is closed. Applications are responsible for checking that all - data has been sent: if only some of the data was transmitted, the - application needs to attempt delivery of the remaining data. - - @param s: data to send. - @type s: str - @return: number of bytes actually sent. - @rtype: int - - @raise socket.timeout: if no data could be sent before the timeout set - by L{settimeout}. - - @since: 1.1 - """ - size = len(s) - self.lock.acquire() - try: - size = self._wait_for_send_window(size) - if size == 0: - # eof or similar - return 0 - m = Message() - m.add_byte(chr(MSG_CHANNEL_EXTENDED_DATA)) - m.add_int(self.remote_chanid) - m.add_int(1) - m.add_string(s[:size]) - finally: - self.lock.release() - # Note: We release self.lock before calling _send_user_message. - # Otherwise, we can deadlock during re-keying. - self.transport._send_user_message(m) - return size - - def sendall(self, s): - """ - Send data to the channel, without allowing partial results. Unlike - L{send}, this method continues to send data from the given string until - either all data has been sent or an error occurs. Nothing is returned. - - @param s: data to send. - @type s: str - - @raise socket.timeout: if sending stalled for longer than the timeout - set by L{settimeout}. - @raise socket.error: if an error occured before the entire string was - sent. - - @note: If the channel is closed while only part of the data hase been - sent, there is no way to determine how much data (if any) was sent. - This is irritating, but identically follows python's API. - """ - while s: - if self.closed: - # this doesn't seem useful, but it is the documented behavior of Socket - raise socket.error('Socket is closed') - sent = self.send(s) - s = s[sent:] - return None - - def sendall_stderr(self, s): - """ - Send data to the channel's "stderr" stream, without allowing partial - results. Unlike L{send_stderr}, this method continues to send data - from the given string until all data has been sent or an error occurs. - Nothing is returned. - - @param s: data to send to the client as "stderr" output. - @type s: str - - @raise socket.timeout: if sending stalled for longer than the timeout - set by L{settimeout}. - @raise socket.error: if an error occured before the entire string was - sent. - - @since: 1.1 - """ - while s: - if self.closed: - raise socket.error('Socket is closed') - sent = self.send_stderr(s) - s = s[sent:] - return None - - def makefile(self, *params): - """ - Return a file-like object associated with this channel. The optional - C{mode} and C{bufsize} arguments are interpreted the same way as by - the built-in C{file()} function in python. - - @return: object which can be used for python file I/O. - @rtype: L{ChannelFile} - """ - return ChannelFile(*([self] + list(params))) - - def makefile_stderr(self, *params): - """ - Return a file-like object associated with this channel's stderr - stream. Only channels using L{exec_command} or L{invoke_shell} - without a pty will ever have data on the stderr stream. - - The optional C{mode} and C{bufsize} arguments are interpreted the - same way as by the built-in C{file()} function in python. For a - client, it only makes sense to open this file for reading. For a - server, it only makes sense to open this file for writing. - - @return: object which can be used for python file I/O. - @rtype: L{ChannelFile} - - @since: 1.1 - """ - return ChannelStderrFile(*([self] + list(params))) - - def fileno(self): - """ - Returns an OS-level file descriptor which can be used for polling, but - but I{not} for reading or writing. This is primaily to allow python's - C{select} module to work. - - The first time C{fileno} is called on a channel, a pipe is created to - simulate real OS-level file descriptor (FD) behavior. Because of this, - two OS-level FDs are created, which will use up FDs faster than normal. - (You won't notice this effect unless you have hundreds of channels - open at the same time.) - - @return: an OS-level file descriptor - @rtype: int - - @warning: This method causes channel reads to be slightly less - efficient. - """ - self.lock.acquire() - try: - if self._pipe is not None: - return self._pipe.fileno() - # create the pipe and feed in any existing data - self._pipe = pipe.make_pipe() - p1, p2 = pipe.make_or_pipe(self._pipe) - self.in_buffer.set_event(p1) - self.in_stderr_buffer.set_event(p2) - return self._pipe.fileno() - finally: - self.lock.release() - - def shutdown(self, how): - """ - Shut down one or both halves of the connection. If C{how} is 0, - further receives are disallowed. If C{how} is 1, further sends - are disallowed. If C{how} is 2, further sends and receives are - disallowed. This closes the stream in one or both directions. - - @param how: 0 (stop receiving), 1 (stop sending), or 2 (stop - receiving and sending). - @type how: int - """ - if (how == 0) or (how == 2): - # feign "read" shutdown - self.eof_received = 1 - if (how == 1) or (how == 2): - self.lock.acquire() - try: - m = self._send_eof() - finally: - self.lock.release() - if m is not None: - self.transport._send_user_message(m) - - def shutdown_read(self): - """ - Shutdown the receiving side of this socket, closing the stream in - the incoming direction. After this call, future reads on this - channel will fail instantly. This is a convenience method, equivalent - to C{shutdown(0)}, for people who don't make it a habit to - memorize unix constants from the 1970s. - - @since: 1.2 - """ - self.shutdown(0) - - def shutdown_write(self): - """ - Shutdown the sending side of this socket, closing the stream in - the outgoing direction. After this call, future writes on this - channel will fail instantly. This is a convenience method, equivalent - to C{shutdown(1)}, for people who don't make it a habit to - memorize unix constants from the 1970s. - - @since: 1.2 - """ - self.shutdown(1) - - - ### calls from Transport - - - def _set_transport(self, transport): - self.transport = transport - self.logger = util.get_logger(self.transport.get_log_channel()) - - def _set_window(self, window_size, max_packet_size): - self.in_window_size = window_size - self.in_max_packet_size = max_packet_size - # threshold of bytes we receive before we bother to send a window update - self.in_window_threshold = window_size // 10 - self.in_window_sofar = 0 - self._log(DEBUG, 'Max packet in: %d bytes' % max_packet_size) - - def _set_remote_channel(self, chanid, window_size, max_packet_size): - self.remote_chanid = chanid - self.out_window_size = window_size - self.out_max_packet_size = max(max_packet_size, MIN_PACKET_SIZE) - self.active = 1 - self._log(DEBUG, 'Max packet out: %d bytes' % max_packet_size) - - def _request_success(self, m): - self._log(DEBUG, 'Sesch channel %d request ok' % self.chanid) - self.event_ready = True - self.event.set() - return - - def _request_failed(self, m): - self.lock.acquire() - try: - msgs = self._close_internal() - finally: - self.lock.release() - for m in msgs: - if m is not None: - self.transport._send_user_message(m) - - def _feed(self, m): - if type(m) is str: - # passed from _feed_extended - s = m - else: - s = m.get_string() - self.in_buffer.feed(s) - - def _feed_extended(self, m): - code = m.get_int() - s = m.get_string() - if code != 1: - self._log(ERROR, 'unknown extended_data type %d; discarding' % code) - return - if self.combine_stderr: - self._feed(s) - else: - self.in_stderr_buffer.feed(s) - - def _window_adjust(self, m): - nbytes = m.get_int() - self.lock.acquire() - try: - if self.ultra_debug: - self._log(DEBUG, 'window up %d' % nbytes) - self.out_window_size += nbytes - self.out_buffer_cv.notifyAll() - finally: - self.lock.release() - - def _handle_request(self, m): - key = m.get_string() - want_reply = m.get_boolean() - server = self.transport.server_object - ok = False - if key == 'exit-status': - self.exit_status = m.get_int() - self.status_event.set() - ok = True - elif key == 'xon-xoff': - # ignore - ok = True - elif key == 'pty-req': - term = m.get_string() - width = m.get_int() - height = m.get_int() - pixelwidth = m.get_int() - pixelheight = m.get_int() - modes = m.get_string() - if server is None: - ok = False - else: - ok = server.check_channel_pty_request(self, term, width, height, pixelwidth, - pixelheight, modes) - elif key == 'shell': - if server is None: - ok = False - else: - ok = server.check_channel_shell_request(self) - elif key == 'env': - name = m.get_string() - value = m.get_string() - if server is None: - ok = False - else: - ok = server.check_channel_env_request(self, name, value) - elif key == 'exec': - cmd = m.get_string() - if server is None: - ok = False - else: - ok = server.check_channel_exec_request(self, cmd) - elif key == 'subsystem': - name = m.get_string() - if server is None: - ok = False - else: - ok = server.check_channel_subsystem_request(self, name) - elif key == 'window-change': - width = m.get_int() - height = m.get_int() - pixelwidth = m.get_int() - pixelheight = m.get_int() - if server is None: - ok = False - else: - ok = server.check_channel_window_change_request(self, width, height, pixelwidth, - pixelheight) - elif key == 'x11-req': - single_connection = m.get_boolean() - auth_proto = m.get_string() - auth_cookie = m.get_string() - screen_number = m.get_int() - if server is None: - ok = False - else: - ok = server.check_channel_x11_request(self, single_connection, - auth_proto, auth_cookie, screen_number) - elif key == 'auth-agent-req@openssh.com': - if server is None: - ok = False - else: - ok = server.check_channel_forward_agent_request(self) - else: - self._log(DEBUG, 'Unhandled channel request "%s"' % key) - ok = False - if want_reply: - m = Message() - if ok: - m.add_byte(chr(MSG_CHANNEL_SUCCESS)) - else: - m.add_byte(chr(MSG_CHANNEL_FAILURE)) - m.add_int(self.remote_chanid) - self.transport._send_user_message(m) - - def _handle_eof(self, m): - self.lock.acquire() - try: - if not self.eof_received: - self.eof_received = True - self.in_buffer.close() - self.in_stderr_buffer.close() - if self._pipe is not None: - self._pipe.set_forever() - finally: - self.lock.release() - self._log(DEBUG, 'EOF received (%s)', self._name) - - def _handle_close(self, m): - self.lock.acquire() - try: - msgs = self._close_internal() - self.transport._unlink_channel(self.chanid) - finally: - self.lock.release() - for m in msgs: - if m is not None: - self.transport._send_user_message(m) - - - ### internals... - - - def _log(self, level, msg, *args): - self.logger.log(level, "[chan " + self._name + "] " + msg, *args) - - def _event_pending(self): - self.event.clear() - self.event_ready = False - - def _wait_for_event(self): - self.event.wait() - assert self.event.isSet() - if self.event_ready: - return - e = self.transport.get_exception() - if e is None: - e = SSHException('Channel closed.') - raise e - - def _set_closed(self): - # you are holding the lock. - self.closed = True - self.in_buffer.close() - self.in_stderr_buffer.close() - self.out_buffer_cv.notifyAll() - # Notify any waiters that we are closed - self.event.set() - self.status_event.set() - if self._pipe is not None: - self._pipe.set_forever() - - def _send_eof(self): - # you are holding the lock. - if self.eof_sent: - return None - m = Message() - m.add_byte(chr(MSG_CHANNEL_EOF)) - m.add_int(self.remote_chanid) - self.eof_sent = True - self._log(DEBUG, 'EOF sent (%s)', self._name) - return m - - def _close_internal(self): - # you are holding the lock. - if not self.active or self.closed: - return None, None - m1 = self._send_eof() - m2 = Message() - m2.add_byte(chr(MSG_CHANNEL_CLOSE)) - m2.add_int(self.remote_chanid) - self._set_closed() - # can't unlink from the Transport yet -- the remote side may still - # try to send meta-data (exit-status, etc) - return m1, m2 - - def _unlink(self): - # server connection could die before we become active: still signal the close! - if self.closed: - return - self.lock.acquire() - try: - self._set_closed() - self.transport._unlink_channel(self.chanid) - finally: - self.lock.release() - - def _check_add_window(self, n): - self.lock.acquire() - try: - if self.closed or self.eof_received or not self.active: - return 0 - if self.ultra_debug: - self._log(DEBUG, 'addwindow %d' % n) - self.in_window_sofar += n - if self.in_window_sofar <= self.in_window_threshold: - return 0 - if self.ultra_debug: - self._log(DEBUG, 'addwindow send %d' % self.in_window_sofar) - out = self.in_window_sofar - self.in_window_sofar = 0 - return out - finally: - self.lock.release() - - def _wait_for_send_window(self, size): - """ - (You are already holding the lock.) - Wait for the send window to open up, and allocate up to C{size} bytes - for transmission. If no space opens up before the timeout, a timeout - exception is raised. Returns the number of bytes available to send - (may be less than requested). - """ - # you are already holding the lock - if self.closed or self.eof_sent: - return 0 - if self.out_window_size == 0: - # should we block? - if self.timeout == 0.0: - raise socket.timeout() - # loop here in case we get woken up but a different thread has filled the buffer - timeout = self.timeout - while self.out_window_size == 0: - if self.closed or self.eof_sent: - return 0 - then = time.time() - self.out_buffer_cv.wait(timeout) - if timeout != None: - timeout -= time.time() - then - if timeout <= 0.0: - raise socket.timeout() - # we have some window to squeeze into - if self.closed or self.eof_sent: - return 0 - if self.out_window_size < size: - size = self.out_window_size - if self.out_max_packet_size - 64 < size: - size = self.out_max_packet_size - 64 - self.out_window_size -= size - if self.ultra_debug: - self._log(DEBUG, 'window down to %d' % self.out_window_size) - return size - - -class ChannelFile (BufferedFile): - """ - A file-like wrapper around L{Channel}. A ChannelFile is created by calling - L{Channel.makefile}. - - @bug: To correctly emulate the file object created from a socket's - C{makefile} method, a L{Channel} and its C{ChannelFile} should be able - to be closed or garbage-collected independently. Currently, closing - the C{ChannelFile} does nothing but flush the buffer. - """ - - def __init__(self, channel, mode = 'r', bufsize = -1): - self.channel = channel - BufferedFile.__init__(self) - self._set_mode(mode, bufsize) - - def __repr__(self): - """ - Returns a string representation of this object, for debugging. - - @rtype: str - """ - return '' - - def _read(self, size): - return self.channel.recv(size) - - def _write(self, data): - self.channel.sendall(data) - return len(data) - - -class ChannelStderrFile (ChannelFile): - def __init__(self, channel, mode = 'r', bufsize = -1): - ChannelFile.__init__(self, channel, mode, bufsize) - - def _read(self, size): - return self.channel.recv_stderr(size) - - def _write(self, data): - self.channel.sendall_stderr(data) - return len(data) - - -# vim: set shiftwidth=4 expandtab : diff --git a/contrib/site-packages/paramiko/client.py b/contrib/site-packages/paramiko/client.py deleted file mode 100644 index c5a2d1ac..00000000 --- a/contrib/site-packages/paramiko/client.py +++ /dev/null @@ -1,538 +0,0 @@ -# Copyright (C) 2006-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -L{SSHClient}. -""" - -from binascii import hexlify -import getpass -import os -import socket -import warnings - -from paramiko.agent import Agent -from paramiko.common import * -from paramiko.config import SSH_PORT -from paramiko.dsskey import DSSKey -from paramiko.hostkeys import HostKeys -from paramiko.resource import ResourceManager -from paramiko.rsakey import RSAKey -from paramiko.ssh_exception import SSHException, BadHostKeyException -from paramiko.transport import Transport -from paramiko.util import retry_on_signal - - -class MissingHostKeyPolicy (object): - """ - Interface for defining the policy that L{SSHClient} should use when the - SSH server's hostname is not in either the system host keys or the - application's keys. Pre-made classes implement policies for automatically - adding the key to the application's L{HostKeys} object (L{AutoAddPolicy}), - and for automatically rejecting the key (L{RejectPolicy}). - - This function may be used to ask the user to verify the key, for example. - """ - - def missing_host_key(self, client, hostname, key): - """ - Called when an L{SSHClient} receives a server key for a server that - isn't in either the system or local L{HostKeys} object. To accept - the key, simply return. To reject, raised an exception (which will - be passed to the calling application). - """ - pass - - -class AutoAddPolicy (MissingHostKeyPolicy): - """ - Policy for automatically adding the hostname and new host key to the - local L{HostKeys} object, and saving it. This is used by L{SSHClient}. - """ - - def missing_host_key(self, client, hostname, key): - client._host_keys.add(hostname, key.get_name(), key) - if client._host_keys_filename is not None: - client.save_host_keys(client._host_keys_filename) - client._log(DEBUG, 'Adding %s host key for %s: %s' % - (key.get_name(), hostname, hexlify(key.get_fingerprint()))) - - -class RejectPolicy (MissingHostKeyPolicy): - """ - Policy for automatically rejecting the unknown hostname & key. This is - used by L{SSHClient}. - """ - - def missing_host_key(self, client, hostname, key): - client._log(DEBUG, 'Rejecting %s host key for %s: %s' % - (key.get_name(), hostname, hexlify(key.get_fingerprint()))) - raise SSHException('Server %r not found in known_hosts' % hostname) - - -class WarningPolicy (MissingHostKeyPolicy): - """ - Policy for logging a python-style warning for an unknown host key, but - accepting it. This is used by L{SSHClient}. - """ - def missing_host_key(self, client, hostname, key): - warnings.warn('Unknown %s host key for %s: %s' % - (key.get_name(), hostname, hexlify(key.get_fingerprint()))) - - -class SSHClient (object): - """ - A high-level representation of a session with an SSH server. This class - wraps L{Transport}, L{Channel}, and L{SFTPClient} to take care of most - aspects of authenticating and opening channels. A typical use case is:: - - client = SSHClient() - client.load_system_host_keys() - client.connect('ssh.example.com') - stdin, stdout, stderr = client.exec_command('ls -l') - - You may pass in explicit overrides for authentication and server host key - checking. The default mechanism is to try to use local key files or an - SSH agent (if one is running). - - @since: 1.6 - """ - - def __init__(self): - """ - Create a new SSHClient. - """ - self._system_host_keys = HostKeys() - self._host_keys = HostKeys() - self._host_keys_filename = None - self._log_channel = None - self._policy = RejectPolicy() - self._transport = None - self._agent = None - - def load_system_host_keys(self, filename=None): - """ - Load host keys from a system (read-only) file. Host keys read with - this method will not be saved back by L{save_host_keys}. - - This method can be called multiple times. Each new set of host keys - will be merged with the existing set (new replacing old if there are - conflicts). - - If C{filename} is left as C{None}, an attempt will be made to read - keys from the user's local "known hosts" file, as used by OpenSSH, - and no exception will be raised if the file can't be read. This is - probably only useful on posix. - - @param filename: the filename to read, or C{None} - @type filename: str - - @raise IOError: if a filename was provided and the file could not be - read - """ - if filename is None: - # try the user's .ssh key file, and mask exceptions - filename = os.path.expanduser('~/.ssh/known_hosts') - try: - self._system_host_keys.load(filename) - except IOError: - pass - return - self._system_host_keys.load(filename) - - def load_host_keys(self, filename): - """ - Load host keys from a local host-key file. Host keys read with this - method will be checked I{after} keys loaded via L{load_system_host_keys}, - but will be saved back by L{save_host_keys} (so they can be modified). - The missing host key policy L{AutoAddPolicy} adds keys to this set and - saves them, when connecting to a previously-unknown server. - - This method can be called multiple times. Each new set of host keys - will be merged with the existing set (new replacing old if there are - conflicts). When automatically saving, the last hostname is used. - - @param filename: the filename to read - @type filename: str - - @raise IOError: if the filename could not be read - """ - self._host_keys_filename = filename - self._host_keys.load(filename) - - def save_host_keys(self, filename): - """ - Save the host keys back to a file. Only the host keys loaded with - L{load_host_keys} (plus any added directly) will be saved -- not any - host keys loaded with L{load_system_host_keys}. - - @param filename: the filename to save to - @type filename: str - - @raise IOError: if the file could not be written - """ - - # update local host keys from file (in case other SSH clients - # have written to the known_hosts file meanwhile. - if self.known_hosts is not None: - self.load_host_keys(self.known_hosts) - - f = open(filename, 'w') - for hostname, keys in self._host_keys.iteritems(): - for keytype, key in keys.iteritems(): - f.write('%s %s %s\n' % (hostname, keytype, key.get_base64())) - f.close() - - def get_host_keys(self): - """ - Get the local L{HostKeys} object. This can be used to examine the - local host keys or change them. - - @return: the local host keys - @rtype: L{HostKeys} - """ - return self._host_keys - - def set_log_channel(self, name): - """ - Set the channel for logging. The default is C{"paramiko.transport"} - but it can be set to anything you want. - - @param name: new channel name for logging - @type name: str - """ - self._log_channel = name - - def set_missing_host_key_policy(self, policy): - """ - Set the policy to use when connecting to a server that doesn't have a - host key in either the system or local L{HostKeys} objects. The - default policy is to reject all unknown servers (using L{RejectPolicy}). - You may substitute L{AutoAddPolicy} or write your own policy class. - - @param policy: the policy to use when receiving a host key from a - previously-unknown server - @type policy: L{MissingHostKeyPolicy} - """ - self._policy = policy - - def connect(self, hostname, port=SSH_PORT, username=None, password=None, pkey=None, - key_filename=None, timeout=None, allow_agent=True, look_for_keys=True, - compress=False, sock=None): - """ - Connect to an SSH server and authenticate to it. The server's host key - is checked against the system host keys (see L{load_system_host_keys}) - and any local host keys (L{load_host_keys}). If the server's hostname - is not found in either set of host keys, the missing host key policy - is used (see L{set_missing_host_key_policy}). The default policy is - to reject the key and raise an L{SSHException}. - - Authentication is attempted in the following order of priority: - - - The C{pkey} or C{key_filename} passed in (if any) - - Any key we can find through an SSH agent - - Any "id_rsa" or "id_dsa" key discoverable in C{~/.ssh/} - - Plain username/password auth, if a password was given - - If a private key requires a password to unlock it, and a password is - passed in, that password will be used to attempt to unlock the key. - - @param hostname: the server to connect to - @type hostname: str - @param port: the server port to connect to - @type port: int - @param username: the username to authenticate as (defaults to the - current local username) - @type username: str - @param password: a password to use for authentication or for unlocking - a private key - @type password: str - @param pkey: an optional private key to use for authentication - @type pkey: L{PKey} - @param key_filename: the filename, or list of filenames, of optional - private key(s) to try for authentication - @type key_filename: str or list(str) - @param timeout: an optional timeout (in seconds) for the TCP connect - @type timeout: float - @param allow_agent: set to False to disable connecting to the SSH agent - @type allow_agent: bool - @param look_for_keys: set to False to disable searching for discoverable - private key files in C{~/.ssh/} - @type look_for_keys: bool - @param compress: set to True to turn on compression - @type compress: bool - @param sock: an open socket or socket-like object (such as a - L{Channel}) to use for communication to the target host - @type sock: socket - - @raise BadHostKeyException: if the server's host key could not be - verified - @raise AuthenticationException: if authentication failed - @raise SSHException: if there was any other error connecting or - establishing an SSH session - @raise socket.error: if a socket error occurred while connecting - """ - if not sock: - for (family, socktype, proto, canonname, sockaddr) in socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM): - if socktype == socket.SOCK_STREAM: - af = family - addr = sockaddr - break - else: - # some OS like AIX don't indicate SOCK_STREAM support, so just guess. :( - af, _, _, _, addr = socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM) - sock = socket.socket(af, socket.SOCK_STREAM) - if timeout is not None: - try: - sock.settimeout(timeout) - except: - pass - retry_on_signal(lambda: sock.connect(addr)) - - t = self._transport = Transport(sock) - t.use_compression(compress=compress) - if self._log_channel is not None: - t.set_log_channel(self._log_channel) - t.start_client() - ResourceManager.register(self, t) - - server_key = t.get_remote_server_key() - keytype = server_key.get_name() - - if port == SSH_PORT: - server_hostkey_name = hostname - else: - server_hostkey_name = "[%s]:%d" % (hostname, port) - our_server_key = self._system_host_keys.get(server_hostkey_name, {}).get(keytype, None) - if our_server_key is None: - our_server_key = self._host_keys.get(server_hostkey_name, {}).get(keytype, None) - if our_server_key is None: - # will raise exception if the key is rejected; let that fall out - self._policy.missing_host_key(self, server_hostkey_name, server_key) - # if the callback returns, assume the key is ok - our_server_key = server_key - - if server_key != our_server_key: - raise BadHostKeyException(hostname, server_key, our_server_key) - - if username is None: - username = getpass.getuser() - - if key_filename is None: - key_filenames = [] - elif isinstance(key_filename, (str, unicode)): - key_filenames = [ key_filename ] - else: - key_filenames = key_filename - self._auth(username, password, pkey, key_filenames, allow_agent, look_for_keys) - - def close(self): - """ - Close this SSHClient and its underlying L{Transport}. - """ - if self._transport is None: - return - self._transport.close() - self._transport = None - - if self._agent != None: - self._agent.close() - self._agent = None - - def exec_command(self, command, bufsize=-1, timeout=None, get_pty=False): - """ - Execute a command on the SSH server. A new L{Channel} is opened and - the requested command is executed. The command's input and output - streams are returned as python C{file}-like objects representing - stdin, stdout, and stderr. - - @param command: the command to execute - @type command: str - @param bufsize: interpreted the same way as by the built-in C{file()} function in python - @type bufsize: int - @param timeout: set command's channel timeout. See L{Channel.settimeout}.settimeout - @type timeout: int - @return: the stdin, stdout, and stderr of the executing command - @rtype: tuple(L{ChannelFile}, L{ChannelFile}, L{ChannelFile}) - - @raise SSHException: if the server fails to execute the command - """ - chan = self._transport.open_session() - if(get_pty): - chan.get_pty() - chan.settimeout(timeout) - chan.exec_command(command) - stdin = chan.makefile('wb', bufsize) - stdout = chan.makefile('rb', bufsize) - stderr = chan.makefile_stderr('rb', bufsize) - return stdin, stdout, stderr - - def invoke_shell(self, term='vt100', width=80, height=24, width_pixels=0, - height_pixels=0): - """ - Start an interactive shell session on the SSH server. A new L{Channel} - is opened and connected to a pseudo-terminal using the requested - terminal type and size. - - @param term: the terminal type to emulate (for example, C{"vt100"}) - @type term: str - @param width: the width (in characters) of the terminal window - @type width: int - @param height: the height (in characters) of the terminal window - @type height: int - @param width_pixels: the width (in pixels) of the terminal window - @type width_pixels: int - @param height_pixels: the height (in pixels) of the terminal window - @type height_pixels: int - @return: a new channel connected to the remote shell - @rtype: L{Channel} - - @raise SSHException: if the server fails to invoke a shell - """ - chan = self._transport.open_session() - chan.get_pty(term, width, height, width_pixels, height_pixels) - chan.invoke_shell() - return chan - - def open_sftp(self): - """ - Open an SFTP session on the SSH server. - - @return: a new SFTP session object - @rtype: L{SFTPClient} - """ - return self._transport.open_sftp_client() - - def get_transport(self): - """ - Return the underlying L{Transport} object for this SSH connection. - This can be used to perform lower-level tasks, like opening specific - kinds of channels. - - @return: the Transport for this connection - @rtype: L{Transport} - """ - return self._transport - - def _auth(self, username, password, pkey, key_filenames, allow_agent, look_for_keys): - """ - Try, in order: - - - The key passed in, if one was passed in. - - Any key we can find through an SSH agent (if allowed). - - Any "id_rsa" or "id_dsa" key discoverable in ~/.ssh/ (if allowed). - - Plain username/password auth, if a password was given. - - (The password might be needed to unlock a private key, or for - two-factor authentication [for which it is required].) - """ - saved_exception = None - two_factor = False - allowed_types = [] - - if pkey is not None: - try: - self._log(DEBUG, 'Trying SSH key %s' % hexlify(pkey.get_fingerprint())) - allowed_types = self._transport.auth_publickey(username, pkey) - two_factor = (allowed_types == ['password']) - if not two_factor: - return - except SSHException, e: - saved_exception = e - - if not two_factor: - for key_filename in key_filenames: - for pkey_class in (RSAKey, DSSKey): - try: - key = pkey_class.from_private_key_file(key_filename, password) - self._log(DEBUG, 'Trying key %s from %s' % (hexlify(key.get_fingerprint()), key_filename)) - self._transport.auth_publickey(username, key) - two_factor = (allowed_types == ['password']) - if not two_factor: - return - break - except SSHException, e: - saved_exception = e - - if not two_factor and allow_agent: - if self._agent == None: - self._agent = Agent() - - for key in self._agent.get_keys(): - try: - self._log(DEBUG, 'Trying SSH agent key %s' % hexlify(key.get_fingerprint())) - # for 2-factor auth a successfully auth'd key will result in ['password'] - allowed_types = self._transport.auth_publickey(username, key) - two_factor = (allowed_types == ['password']) - if not two_factor: - return - break - except SSHException, e: - saved_exception = e - - if not two_factor: - keyfiles = [] - rsa_key = os.path.expanduser('~/.ssh/id_rsa') - dsa_key = os.path.expanduser('~/.ssh/id_dsa') - if os.path.isfile(rsa_key): - keyfiles.append((RSAKey, rsa_key)) - if os.path.isfile(dsa_key): - keyfiles.append((DSSKey, dsa_key)) - # look in ~/ssh/ for windows users: - rsa_key = os.path.expanduser('~/ssh/id_rsa') - dsa_key = os.path.expanduser('~/ssh/id_dsa') - if os.path.isfile(rsa_key): - keyfiles.append((RSAKey, rsa_key)) - if os.path.isfile(dsa_key): - keyfiles.append((DSSKey, dsa_key)) - - if not look_for_keys: - keyfiles = [] - - for pkey_class, filename in keyfiles: - try: - key = pkey_class.from_private_key_file(filename, password) - self._log(DEBUG, 'Trying discovered key %s in %s' % (hexlify(key.get_fingerprint()), filename)) - # for 2-factor auth a successfully auth'd key will result in ['password'] - allowed_types = self._transport.auth_publickey(username, key) - two_factor = (allowed_types == ['password']) - if not two_factor: - return - break - except SSHException, e: - saved_exception = e - except IOError, e: - saved_exception = e - - if password is not None: - try: - self._transport.auth_password(username, password) - return - except SSHException, e: - saved_exception = e - elif two_factor: - raise SSHException('Two-factor authentication requires a password') - - # if we got an auth-failed exception earlier, re-raise it - if saved_exception is not None: - raise saved_exception - raise SSHException('No authentication methods available') - - def _log(self, level, msg): - self._transport._log(level, msg) - diff --git a/contrib/site-packages/paramiko/common.py b/contrib/site-packages/paramiko/common.py deleted file mode 100644 index 3d7ca588..00000000 --- a/contrib/site-packages/paramiko/common.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -Common constants and global variables. -""" - -MSG_DISCONNECT, MSG_IGNORE, MSG_UNIMPLEMENTED, MSG_DEBUG, MSG_SERVICE_REQUEST, \ - MSG_SERVICE_ACCEPT = range(1, 7) -MSG_KEXINIT, MSG_NEWKEYS = range(20, 22) -MSG_USERAUTH_REQUEST, MSG_USERAUTH_FAILURE, MSG_USERAUTH_SUCCESS, \ - MSG_USERAUTH_BANNER = range(50, 54) -MSG_USERAUTH_PK_OK = 60 -MSG_USERAUTH_INFO_REQUEST, MSG_USERAUTH_INFO_RESPONSE = range(60, 62) -MSG_GLOBAL_REQUEST, MSG_REQUEST_SUCCESS, MSG_REQUEST_FAILURE = range(80, 83) -MSG_CHANNEL_OPEN, MSG_CHANNEL_OPEN_SUCCESS, MSG_CHANNEL_OPEN_FAILURE, \ - MSG_CHANNEL_WINDOW_ADJUST, MSG_CHANNEL_DATA, MSG_CHANNEL_EXTENDED_DATA, \ - MSG_CHANNEL_EOF, MSG_CHANNEL_CLOSE, MSG_CHANNEL_REQUEST, \ - MSG_CHANNEL_SUCCESS, MSG_CHANNEL_FAILURE = range(90, 101) - - -# for debugging: -MSG_NAMES = { - MSG_DISCONNECT: 'disconnect', - MSG_IGNORE: 'ignore', - MSG_UNIMPLEMENTED: 'unimplemented', - MSG_DEBUG: 'debug', - MSG_SERVICE_REQUEST: 'service-request', - MSG_SERVICE_ACCEPT: 'service-accept', - MSG_KEXINIT: 'kexinit', - MSG_NEWKEYS: 'newkeys', - 30: 'kex30', - 31: 'kex31', - 32: 'kex32', - 33: 'kex33', - 34: 'kex34', - MSG_USERAUTH_REQUEST: 'userauth-request', - MSG_USERAUTH_FAILURE: 'userauth-failure', - MSG_USERAUTH_SUCCESS: 'userauth-success', - MSG_USERAUTH_BANNER: 'userauth--banner', - MSG_USERAUTH_PK_OK: 'userauth-60(pk-ok/info-request)', - MSG_USERAUTH_INFO_RESPONSE: 'userauth-info-response', - MSG_GLOBAL_REQUEST: 'global-request', - MSG_REQUEST_SUCCESS: 'request-success', - MSG_REQUEST_FAILURE: 'request-failure', - MSG_CHANNEL_OPEN: 'channel-open', - MSG_CHANNEL_OPEN_SUCCESS: 'channel-open-success', - MSG_CHANNEL_OPEN_FAILURE: 'channel-open-failure', - MSG_CHANNEL_WINDOW_ADJUST: 'channel-window-adjust', - MSG_CHANNEL_DATA: 'channel-data', - MSG_CHANNEL_EXTENDED_DATA: 'channel-extended-data', - MSG_CHANNEL_EOF: 'channel-eof', - MSG_CHANNEL_CLOSE: 'channel-close', - MSG_CHANNEL_REQUEST: 'channel-request', - MSG_CHANNEL_SUCCESS: 'channel-success', - MSG_CHANNEL_FAILURE: 'channel-failure' - } - - -# authentication request return codes: -AUTH_SUCCESSFUL, AUTH_PARTIALLY_SUCCESSFUL, AUTH_FAILED = range(3) - - -# channel request failed reasons: -(OPEN_SUCCEEDED, - OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED, - OPEN_FAILED_CONNECT_FAILED, - OPEN_FAILED_UNKNOWN_CHANNEL_TYPE, - OPEN_FAILED_RESOURCE_SHORTAGE) = range(0, 5) - - -CONNECTION_FAILED_CODE = { - 1: 'Administratively prohibited', - 2: 'Connect failed', - 3: 'Unknown channel type', - 4: 'Resource shortage' -} - - -DISCONNECT_SERVICE_NOT_AVAILABLE, DISCONNECT_AUTH_CANCELLED_BY_USER, \ - DISCONNECT_NO_MORE_AUTH_METHODS_AVAILABLE = 7, 13, 14 - -from Crypto import Random - -# keep a crypto-strong PRNG nearby -rng = Random.new() - -import sys -if sys.version_info < (2, 3): - try: - import logging - except: - import logging22 as logging - import select - PY22 = True - - import socket - if not hasattr(socket, 'timeout'): - class timeout(socket.error): pass - socket.timeout = timeout - del timeout -else: - import logging - PY22 = False - - -DEBUG = logging.DEBUG -INFO = logging.INFO -WARNING = logging.WARNING -ERROR = logging.ERROR -CRITICAL = logging.CRITICAL - -# Common IO/select/etc sleep period, in seconds -io_sleep = 0.01 diff --git a/contrib/site-packages/paramiko/compress.py b/contrib/site-packages/paramiko/compress.py deleted file mode 100644 index b55f0b1d..00000000 --- a/contrib/site-packages/paramiko/compress.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -Compression implementations for a Transport. -""" - -import zlib - - -class ZlibCompressor (object): - def __init__(self): - self.z = zlib.compressobj(9) - - def __call__(self, data): - return self.z.compress(data) + self.z.flush(zlib.Z_FULL_FLUSH) - - -class ZlibDecompressor (object): - def __init__(self): - self.z = zlib.decompressobj() - - def __call__(self, data): - return self.z.decompress(data) diff --git a/contrib/site-packages/paramiko/config.py b/contrib/site-packages/paramiko/config.py deleted file mode 100644 index 1705de76..00000000 --- a/contrib/site-packages/paramiko/config.py +++ /dev/null @@ -1,266 +0,0 @@ -# Copyright (C) 2006-2007 Robey Pointer -# Copyright (C) 2012 Olle Lundberg -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -L{SSHConfig}. -""" - -import fnmatch -import os -import re -import socket - -SSH_PORT = 22 -proxy_re = re.compile(r"^(proxycommand)\s*=*\s*(.*)", re.I) - - -class LazyFqdn(object): - """ - Returns the host's fqdn on request as string. - """ - - def __init__(self, config, host=None): - self.fqdn = None - self.config = config - self.host = host - - def __str__(self): - if self.fqdn is None: - # - # If the SSH config contains AddressFamily, use that when - # determining the local host's FQDN. Using socket.getfqdn() from - # the standard library is the most general solution, but can - # result in noticeable delays on some platforms when IPv6 is - # misconfigured or not available, as it calls getaddrinfo with no - # address family specified, so both IPv4 and IPv6 are checked. - # - - # Handle specific option - fqdn = None - address_family = self.config.get('addressfamily', 'any').lower() - if address_family != 'any': - try: - family = socket.AF_INET if address_family == 'inet' \ - else socket.AF_INET6 - results = socket.getaddrinfo( - self.host, - None, - family, - socket.SOCK_DGRAM, - socket.IPPROTO_IP, - socket.AI_CANONNAME - ) - for res in results: - af, socktype, proto, canonname, sa = res - if canonname and '.' in canonname: - fqdn = canonname - break - # giaerror -> socket.getaddrinfo() can't resolve self.host - # (which is from socket.gethostname()). Fall back to the - # getfqdn() call below. - except socket.gaierror: - pass - # Handle 'any' / unspecified - if fqdn is None: - fqdn = socket.getfqdn() - # Cache - self.fqdn = fqdn - return self.fqdn - - -class SSHConfig (object): - """ - Representation of config information as stored in the format used by - OpenSSH. Queries can be made via L{lookup}. The format is described in - OpenSSH's C{ssh_config} man page. This class is provided primarily as a - convenience to posix users (since the OpenSSH format is a de-facto - standard on posix) but should work fine on Windows too. - - @since: 1.6 - """ - - def __init__(self): - """ - Create a new OpenSSH config object. - """ - self._config = [] - - def parse(self, file_obj): - """ - Read an OpenSSH config from the given file object. - - @param file_obj: a file-like object to read the config file from - @type file_obj: file - """ - host = {"host": ['*'], "config": {}} - for line in file_obj: - line = line.rstrip('\n').lstrip() - if (line == '') or (line[0] == '#'): - continue - if '=' in line: - # Ensure ProxyCommand gets properly split - if line.lower().strip().startswith('proxycommand'): - match = proxy_re.match(line) - key, value = match.group(1).lower(), match.group(2) - else: - key, value = line.split('=', 1) - key = key.strip().lower() - else: - # find first whitespace, and split there - i = 0 - while (i < len(line)) and not line[i].isspace(): - i += 1 - if i == len(line): - raise Exception('Unparsable line: %r' % line) - key = line[:i].lower() - value = line[i:].lstrip() - - if key == 'host': - self._config.append(host) - value = value.split() - host = {key: value, 'config': {}} - #identityfile, localforward, remoteforward keys are special cases, since they are allowed to be - # specified multiple times and they should be tried in order - # of specification. - - elif key in ['identityfile', 'localforward', 'remoteforward']: - if key in host['config']: - host['config'][key].append(value) - else: - host['config'][key] = [value] - elif key not in host['config']: - host['config'].update({key: value}) - self._config.append(host) - - def lookup(self, hostname): - """ - Return a dict of config options for a given hostname. - - The host-matching rules of OpenSSH's C{ssh_config} man page are used, - which means that all configuration options from matching host - specifications are merged, with more specific hostmasks taking - precedence. In other words, if C{"Port"} is set under C{"Host *"} - and also C{"Host *.example.com"}, and the lookup is for - C{"ssh.example.com"}, then the port entry for C{"Host *.example.com"} - will win out. - - The keys in the returned dict are all normalized to lowercase (look for - C{"port"}, not C{"Port"}. The values are processed according to the - rules for substitution variable expansion in C{ssh_config}. - - @param hostname: the hostname to lookup - @type hostname: str - """ - - matches = [config for config in self._config if - self._allowed(hostname, config['host'])] - - ret = {} - for match in matches: - for key, value in match['config'].iteritems(): - if key not in ret: - # Create a copy of the original value, - # else it will reference the original list - # in self._config and update that value too - # when the extend() is being called. - ret[key] = value[:] - elif key == 'identityfile': - ret[key].extend(value) - ret = self._expand_variables(ret, hostname) - return ret - - def _allowed(self, hostname, hosts): - match = False - for host in hosts: - if host.startswith('!') and fnmatch.fnmatch(hostname, host[1:]): - return False - elif fnmatch.fnmatch(hostname, host): - match = True - return match - - def _expand_variables(self, config, hostname): - """ - Return a dict of config options with expanded substitutions - for a given hostname. - - Please refer to man C{ssh_config} for the parameters that - are replaced. - - @param config: the config for the hostname - @type hostname: dict - @param hostname: the hostname that the config belongs to - @type hostname: str - """ - - if 'hostname' in config: - config['hostname'] = config['hostname'].replace('%h', hostname) - else: - config['hostname'] = hostname - - if 'port' in config: - port = config['port'] - else: - port = SSH_PORT - - user = os.getenv('USER') - if 'user' in config: - remoteuser = config['user'] - else: - remoteuser = user - - host = socket.gethostname().split('.')[0] - fqdn = LazyFqdn(config, host) - homedir = os.path.expanduser('~') - replacements = {'controlpath': - [ - ('%h', config['hostname']), - ('%l', fqdn), - ('%L', host), - ('%n', hostname), - ('%p', port), - ('%r', remoteuser), - ('%u', user) - ], - 'identityfile': - [ - ('~', homedir), - ('%d', homedir), - ('%h', config['hostname']), - ('%l', fqdn), - ('%u', user), - ('%r', remoteuser) - ], - 'proxycommand': - [ - ('%h', config['hostname']), - ('%p', port), - ('%r', remoteuser) - ] - } - - for k in config: - if k in replacements: - for find, replace in replacements[k]: - if isinstance(config[k], list): - for item in range(len(config[k])): - config[k][item] = config[k][item].\ - replace(find, str(replace)) - else: - config[k] = config[k].replace(find, str(replace)) - return config diff --git a/contrib/site-packages/paramiko/dsskey.py b/contrib/site-packages/paramiko/dsskey.py deleted file mode 100644 index f6ecb2a7..00000000 --- a/contrib/site-packages/paramiko/dsskey.py +++ /dev/null @@ -1,196 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -L{DSSKey} -""" - -from Crypto.PublicKey import DSA -from Crypto.Hash import SHA - -from paramiko.common import * -from paramiko import util -from paramiko.ssh_exception import SSHException -from paramiko.message import Message -from paramiko.ber import BER, BERException -from paramiko.pkey import PKey - - -class DSSKey (PKey): - """ - Representation of a DSS key which can be used to sign an verify SSH2 - data. - """ - - def __init__(self, msg=None, data=None, filename=None, password=None, vals=None, file_obj=None): - self.p = None - self.q = None - self.g = None - self.y = None - self.x = None - if file_obj is not None: - self._from_private_key(file_obj, password) - return - if filename is not None: - self._from_private_key_file(filename, password) - return - if (msg is None) and (data is not None): - msg = Message(data) - if vals is not None: - self.p, self.q, self.g, self.y = vals - else: - if msg is None: - raise SSHException('Key object may not be empty') - if msg.get_string() != 'ssh-dss': - raise SSHException('Invalid key') - self.p = msg.get_mpint() - self.q = msg.get_mpint() - self.g = msg.get_mpint() - self.y = msg.get_mpint() - self.size = util.bit_length(self.p) - - def __str__(self): - m = Message() - m.add_string('ssh-dss') - m.add_mpint(self.p) - m.add_mpint(self.q) - m.add_mpint(self.g) - m.add_mpint(self.y) - return str(m) - - def __hash__(self): - h = hash(self.get_name()) - h = h * 37 + hash(self.p) - h = h * 37 + hash(self.q) - h = h * 37 + hash(self.g) - h = h * 37 + hash(self.y) - # h might be a long by now... - return hash(h) - - def get_name(self): - return 'ssh-dss' - - def get_bits(self): - return self.size - - def can_sign(self): - return self.x is not None - - def sign_ssh_data(self, rng, data): - digest = SHA.new(data).digest() - dss = DSA.construct((long(self.y), long(self.g), long(self.p), long(self.q), long(self.x))) - # generate a suitable k - qsize = len(util.deflate_long(self.q, 0)) - while True: - k = util.inflate_long(rng.read(qsize), 1) - if (k > 2) and (k < self.q): - break - r, s = dss.sign(util.inflate_long(digest, 1), k) - m = Message() - m.add_string('ssh-dss') - # apparently, in rare cases, r or s may be shorter than 20 bytes! - rstr = util.deflate_long(r, 0) - sstr = util.deflate_long(s, 0) - if len(rstr) < 20: - rstr = '\x00' * (20 - len(rstr)) + rstr - if len(sstr) < 20: - sstr = '\x00' * (20 - len(sstr)) + sstr - m.add_string(rstr + sstr) - return m - - def verify_ssh_sig(self, data, msg): - if len(str(msg)) == 40: - # spies.com bug: signature has no header - sig = str(msg) - else: - kind = msg.get_string() - if kind != 'ssh-dss': - return 0 - sig = msg.get_string() - - # pull out (r, s) which are NOT encoded as mpints - sigR = util.inflate_long(sig[:20], 1) - sigS = util.inflate_long(sig[20:], 1) - sigM = util.inflate_long(SHA.new(data).digest(), 1) - - dss = DSA.construct((long(self.y), long(self.g), long(self.p), long(self.q))) - return dss.verify(sigM, (sigR, sigS)) - - def _encode_key(self): - if self.x is None: - raise SSHException('Not enough key information') - keylist = [ 0, self.p, self.q, self.g, self.y, self.x ] - try: - b = BER() - b.encode(keylist) - except BERException: - raise SSHException('Unable to create ber encoding of key') - return str(b) - - def write_private_key_file(self, filename, password=None): - self._write_private_key_file('DSA', filename, self._encode_key(), password) - - def write_private_key(self, file_obj, password=None): - self._write_private_key('DSA', file_obj, self._encode_key(), password) - - def generate(bits=1024, progress_func=None): - """ - Generate a new private DSS key. This factory function can be used to - generate a new host key or authentication key. - - @param bits: number of bits the generated key should be. - @type bits: int - @param progress_func: an optional function to call at key points in - key generation (used by C{pyCrypto.PublicKey}). - @type progress_func: function - @return: new private key - @rtype: L{DSSKey} - """ - dsa = DSA.generate(bits, rng.read, progress_func) - key = DSSKey(vals=(dsa.p, dsa.q, dsa.g, dsa.y)) - key.x = dsa.x - return key - generate = staticmethod(generate) - - - ### internals... - - - def _from_private_key_file(self, filename, password): - data = self._read_private_key_file('DSA', filename, password) - self._decode_key(data) - - def _from_private_key(self, file_obj, password): - data = self._read_private_key('DSA', file_obj, password) - self._decode_key(data) - - def _decode_key(self, data): - # private key file contains: - # DSAPrivateKey = { version = 0, p, q, g, y, x } - try: - keylist = BER(data).decode() - except BERException, x: - raise SSHException('Unable to parse key file: ' + str(x)) - if (type(keylist) is not list) or (len(keylist) < 6) or (keylist[0] != 0): - raise SSHException('not a valid DSA private key file (bad ber encoding)') - self.p = keylist[1] - self.q = keylist[2] - self.g = keylist[3] - self.y = keylist[4] - self.x = keylist[5] - self.size = util.bit_length(self.p) diff --git a/contrib/site-packages/paramiko/ecdsakey.py b/contrib/site-packages/paramiko/ecdsakey.py deleted file mode 100644 index ac840ab7..00000000 --- a/contrib/site-packages/paramiko/ecdsakey.py +++ /dev/null @@ -1,181 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distrubuted in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -L{ECDSAKey} -""" - -import binascii -from ecdsa import SigningKey, VerifyingKey, der, curves -from ecdsa.util import number_to_string, sigencode_string, sigencode_strings, sigdecode_strings -from Crypto.Hash import SHA256, MD5 -from Crypto.Cipher import DES3 - -from paramiko.common import * -from paramiko import util -from paramiko.message import Message -from paramiko.ber import BER, BERException -from paramiko.pkey import PKey -from paramiko.ssh_exception import SSHException - - -class ECDSAKey (PKey): - """ - Representation of an ECDSA key which can be used to sign and verify SSH2 - data. - """ - - def __init__(self, msg=None, data=None, filename=None, password=None, vals=None, file_obj=None): - self.verifying_key = None - self.signing_key = None - if file_obj is not None: - self._from_private_key(file_obj, password) - return - if filename is not None: - self._from_private_key_file(filename, password) - return - if (msg is None) and (data is not None): - msg = Message(data) - if vals is not None: - self.verifying_key, self.signing_key = vals - else: - if msg is None: - raise SSHException('Key object may not be empty') - if msg.get_string() != 'ecdsa-sha2-nistp256': - raise SSHException('Invalid key') - curvename = msg.get_string() - if curvename != 'nistp256': - raise SSHException("Can't handle curve of type %s" % curvename) - - pointinfo = msg.get_string() - if pointinfo[0] != "\x04": - raise SSHException('Point compression is being used: %s'% - binascii.hexlify(pointinfo)) - self.verifying_key = VerifyingKey.from_string(pointinfo[1:], - curve=curves.NIST256p) - self.size = 256 - - def __str__(self): - key = self.verifying_key - m = Message() - m.add_string('ecdsa-sha2-nistp256') - m.add_string('nistp256') - - point_str = "\x04" + key.to_string() - - m.add_string(point_str) - return str(m) - - def __hash__(self): - h = hash(self.get_name()) - h = h * 37 + hash(self.verifying_key.pubkey.point.x()) - h = h * 37 + hash(self.verifying_key.pubkey.point.y()) - return hash(h) - - def get_name(self): - return 'ecdsa-sha2-nistp256' - - def get_bits(self): - return self.size - - def can_sign(self): - return self.signing_key is not None - - def sign_ssh_data(self, rpool, data): - digest = SHA256.new(data).digest() - sig = self.signing_key.sign_digest(digest, entropy=rpool.read, - sigencode=self._sigencode) - m = Message() - m.add_string('ecdsa-sha2-nistp256') - m.add_string(sig) - return m - - def verify_ssh_sig(self, data, msg): - if msg.get_string() != 'ecdsa-sha2-nistp256': - return False - sig = msg.get_string() - - # verify the signature by SHA'ing the data and encrypting it - # using the public key. - hash_obj = SHA256.new(data).digest() - return self.verifying_key.verify_digest(sig, hash_obj, - sigdecode=self._sigdecode) - - def write_private_key_file(self, filename, password=None): - key = self.signing_key or self.verifying_key - self._write_private_key_file('EC', filename, key.to_der(), password) - - def write_private_key(self, file_obj, password=None): - key = self.signing_key or self.verifying_key - self._write_private_key('EC', file_obj, key.to_der(), password) - - def generate(bits, progress_func=None): - """ - Generate a new private RSA key. This factory function can be used to - generate a new host key or authentication key. - - @param bits: number of bits the generated key should be. - @type bits: int - @param progress_func: an optional function to call at key points in - key generation (used by C{pyCrypto.PublicKey}). - @type progress_func: function - @return: new private key - @rtype: L{RSAKey} - """ - signing_key = ECDSA.generate() - key = ECDSAKey(vals=(signing_key, signing_key.get_verifying_key())) - return key - generate = staticmethod(generate) - - - ### internals... - - - def _from_private_key_file(self, filename, password): - data = self._read_private_key_file('EC', filename, password) - self._decode_key(data) - - def _from_private_key(self, file_obj, password): - data = self._read_private_key('EC', file_obj, password) - self._decode_key(data) - - ALLOWED_PADDINGS = ['\x01', '\x02\x02', '\x03\x03\x03', '\x04\x04\x04\x04', - '\x05\x05\x05\x05\x05', '\x06\x06\x06\x06\x06\x06', - '\x07\x07\x07\x07\x07\x07\x07'] - def _decode_key(self, data): - s, padding = der.remove_sequence(data) - if padding: - if padding not in self.ALLOWED_PADDINGS: - raise ValueError, "weird padding: %s" % (binascii.hexlify(empty)) - data = data[:-len(padding)] - key = SigningKey.from_der(data) - self.signing_key = key - self.verifying_key = key.get_verifying_key() - self.size = 256 - - def _sigencode(self, r, s, order): - msg = Message() - msg.add_mpint(r) - msg.add_mpint(s) - return str(msg) - - def _sigdecode(self, sig, order): - msg = Message(sig) - r = msg.get_mpint() - s = msg.get_mpint() - return (r, s) diff --git a/contrib/site-packages/paramiko/file.py b/contrib/site-packages/paramiko/file.py deleted file mode 100644 index 5fd81cfe..00000000 --- a/contrib/site-packages/paramiko/file.py +++ /dev/null @@ -1,460 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -BufferedFile. -""" - -from cStringIO import StringIO - - -class BufferedFile (object): - """ - Reusable base class to implement python-style file buffering around a - simpler stream. - """ - - _DEFAULT_BUFSIZE = 8192 - - SEEK_SET = 0 - SEEK_CUR = 1 - SEEK_END = 2 - - FLAG_READ = 0x1 - FLAG_WRITE = 0x2 - FLAG_APPEND = 0x4 - FLAG_BINARY = 0x10 - FLAG_BUFFERED = 0x20 - FLAG_LINE_BUFFERED = 0x40 - FLAG_UNIVERSAL_NEWLINE = 0x80 - - def __init__(self): - self.newlines = None - self._flags = 0 - self._bufsize = self._DEFAULT_BUFSIZE - self._wbuffer = StringIO() - self._rbuffer = '' - self._at_trailing_cr = False - self._closed = False - # pos - position within the file, according to the user - # realpos - position according the OS - # (these may be different because we buffer for line reading) - self._pos = self._realpos = 0 - # size only matters for seekable files - self._size = 0 - - def __del__(self): - self.close() - - def __iter__(self): - """ - Returns an iterator that can be used to iterate over the lines in this - file. This iterator happens to return the file itself, since a file is - its own iterator. - - @raise ValueError: if the file is closed. - - @return: an interator. - @rtype: iterator - """ - if self._closed: - raise ValueError('I/O operation on closed file') - return self - - def close(self): - """ - Close the file. Future read and write operations will fail. - """ - self.flush() - self._closed = True - - def flush(self): - """ - Write out any data in the write buffer. This may do nothing if write - buffering is not turned on. - """ - self._write_all(self._wbuffer.getvalue()) - self._wbuffer = StringIO() - return - - def next(self): - """ - Returns the next line from the input, or raises L{StopIteration} when - EOF is hit. Unlike python file objects, it's okay to mix calls to - C{next} and L{readline}. - - @raise StopIteration: when the end of the file is reached. - - @return: a line read from the file. - @rtype: str - """ - line = self.readline() - if not line: - raise StopIteration - return line - - def read(self, size=None): - """ - Read at most C{size} bytes from the file (less if we hit the end of the - file first). If the C{size} argument is negative or omitted, read all - the remaining data in the file. - - @param size: maximum number of bytes to read - @type size: int - @return: data read from the file, or an empty string if EOF was - encountered immediately - @rtype: str - """ - if self._closed: - raise IOError('File is closed') - if not (self._flags & self.FLAG_READ): - raise IOError('File is not open for reading') - if (size is None) or (size < 0): - # go for broke - result = self._rbuffer - self._rbuffer = '' - self._pos += len(result) - while True: - try: - new_data = self._read(self._DEFAULT_BUFSIZE) - except EOFError: - new_data = None - if (new_data is None) or (len(new_data) == 0): - break - result += new_data - self._realpos += len(new_data) - self._pos += len(new_data) - return result - if size <= len(self._rbuffer): - result = self._rbuffer[:size] - self._rbuffer = self._rbuffer[size:] - self._pos += len(result) - return result - while len(self._rbuffer) < size: - read_size = size - len(self._rbuffer) - if self._flags & self.FLAG_BUFFERED: - read_size = max(self._bufsize, read_size) - try: - new_data = self._read(read_size) - except EOFError: - new_data = None - if (new_data is None) or (len(new_data) == 0): - break - self._rbuffer += new_data - self._realpos += len(new_data) - result = self._rbuffer[:size] - self._rbuffer = self._rbuffer[size:] - self._pos += len(result) - return result - - def readline(self, size=None): - """ - Read one entire line from the file. A trailing newline character is - kept in the string (but may be absent when a file ends with an - incomplete line). If the size argument is present and non-negative, it - is a maximum byte count (including the trailing newline) and an - incomplete line may be returned. An empty string is returned only when - EOF is encountered immediately. - - @note: Unlike stdio's C{fgets()}, the returned string contains null - characters (C{'\\0'}) if they occurred in the input. - - @param size: maximum length of returned string. - @type size: int - @return: next line of the file, or an empty string if the end of the - file has been reached. - @rtype: str - """ - # it's almost silly how complex this function is. - if self._closed: - raise IOError('File is closed') - if not (self._flags & self.FLAG_READ): - raise IOError('File not open for reading') - line = self._rbuffer - while True: - if self._at_trailing_cr and (self._flags & self.FLAG_UNIVERSAL_NEWLINE) and (len(line) > 0): - # edge case: the newline may be '\r\n' and we may have read - # only the first '\r' last time. - if line[0] == '\n': - line = line[1:] - self._record_newline('\r\n') - else: - self._record_newline('\r') - self._at_trailing_cr = False - # check size before looking for a linefeed, in case we already have - # enough. - if (size is not None) and (size >= 0): - if len(line) >= size: - # truncate line and return - self._rbuffer = line[size:] - line = line[:size] - self._pos += len(line) - return line - n = size - len(line) - else: - n = self._bufsize - if ('\n' in line) or ((self._flags & self.FLAG_UNIVERSAL_NEWLINE) and ('\r' in line)): - break - try: - new_data = self._read(n) - except EOFError: - new_data = None - if (new_data is None) or (len(new_data) == 0): - self._rbuffer = '' - self._pos += len(line) - return line - line += new_data - self._realpos += len(new_data) - # find the newline - pos = line.find('\n') - if self._flags & self.FLAG_UNIVERSAL_NEWLINE: - rpos = line.find('\r') - if (rpos >= 0) and ((rpos < pos) or (pos < 0)): - pos = rpos - xpos = pos + 1 - if (line[pos] == '\r') and (xpos < len(line)) and (line[xpos] == '\n'): - xpos += 1 - self._rbuffer = line[xpos:] - lf = line[pos:xpos] - line = line[:pos] + '\n' - if (len(self._rbuffer) == 0) and (lf == '\r'): - # we could read the line up to a '\r' and there could still be a - # '\n' following that we read next time. note that and eat it. - self._at_trailing_cr = True - else: - self._record_newline(lf) - self._pos += len(line) - return line - - def readlines(self, sizehint=None): - """ - Read all remaining lines using L{readline} and return them as a list. - If the optional C{sizehint} argument is present, instead of reading up - to EOF, whole lines totalling approximately sizehint bytes (possibly - after rounding up to an internal buffer size) are read. - - @param sizehint: desired maximum number of bytes to read. - @type sizehint: int - @return: list of lines read from the file. - @rtype: list - """ - lines = [] - bytes = 0 - while True: - line = self.readline() - if len(line) == 0: - break - lines.append(line) - bytes += len(line) - if (sizehint is not None) and (bytes >= sizehint): - break - return lines - - def seek(self, offset, whence=0): - """ - Set the file's current position, like stdio's C{fseek}. Not all file - objects support seeking. - - @note: If a file is opened in append mode (C{'a'} or C{'a+'}), any seek - operations will be undone at the next write (as the file position - will move back to the end of the file). - - @param offset: position to move to within the file, relative to - C{whence}. - @type offset: int - @param whence: type of movement: 0 = absolute; 1 = relative to the - current position; 2 = relative to the end of the file. - @type whence: int - - @raise IOError: if the file doesn't support random access. - """ - raise IOError('File does not support seeking.') - - def tell(self): - """ - Return the file's current position. This may not be accurate or - useful if the underlying file doesn't support random access, or was - opened in append mode. - - @return: file position (in bytes). - @rtype: int - """ - return self._pos - - def write(self, data): - """ - Write data to the file. If write buffering is on (C{bufsize} was - specified and non-zero), some or all of the data may not actually be - written yet. (Use L{flush} or L{close} to force buffered data to be - written out.) - - @param data: data to write. - @type data: str - """ - if self._closed: - raise IOError('File is closed') - if not (self._flags & self.FLAG_WRITE): - raise IOError('File not open for writing') - if not (self._flags & self.FLAG_BUFFERED): - self._write_all(data) - return - self._wbuffer.write(data) - if self._flags & self.FLAG_LINE_BUFFERED: - # only scan the new data for linefeed, to avoid wasting time. - last_newline_pos = data.rfind('\n') - if last_newline_pos >= 0: - wbuf = self._wbuffer.getvalue() - last_newline_pos += len(wbuf) - len(data) - self._write_all(wbuf[:last_newline_pos + 1]) - self._wbuffer = StringIO() - self._wbuffer.write(wbuf[last_newline_pos + 1:]) - return - # even if we're line buffering, if the buffer has grown past the - # buffer size, force a flush. - if self._wbuffer.tell() >= self._bufsize: - self.flush() - return - - def writelines(self, sequence): - """ - Write a sequence of strings to the file. The sequence can be any - iterable object producing strings, typically a list of strings. (The - name is intended to match L{readlines}; C{writelines} does not add line - separators.) - - @param sequence: an iterable sequence of strings. - @type sequence: sequence - """ - for line in sequence: - self.write(line) - return - - def xreadlines(self): - """ - Identical to C{iter(f)}. This is a deprecated file interface that - predates python iterator support. - - @return: an iterator. - @rtype: iterator - """ - return self - - @property - def closed(self): - return self._closed - - - ### overrides... - - - def _read(self, size): - """ - I{(subclass override)} - Read data from the stream. Return C{None} or raise C{EOFError} to - indicate EOF. - """ - raise EOFError() - - def _write(self, data): - """ - I{(subclass override)} - Write data into the stream. - """ - raise IOError('write not implemented') - - def _get_size(self): - """ - I{(subclass override)} - Return the size of the file. This is called from within L{_set_mode} - if the file is opened in append mode, so the file position can be - tracked and L{seek} and L{tell} will work correctly. If the file is - a stream that can't be randomly accessed, you don't need to override - this method, - """ - return 0 - - - ### internals... - - - def _set_mode(self, mode='r', bufsize=-1): - """ - Subclasses call this method to initialize the BufferedFile. - """ - # set bufsize in any event, because it's used for readline(). - self._bufsize = self._DEFAULT_BUFSIZE - if bufsize < 0: - # do no buffering by default, because otherwise writes will get - # buffered in a way that will probably confuse people. - bufsize = 0 - if bufsize == 1: - # apparently, line buffering only affects writes. reads are only - # buffered if you call readline (directly or indirectly: iterating - # over a file will indirectly call readline). - self._flags |= self.FLAG_BUFFERED | self.FLAG_LINE_BUFFERED - elif bufsize > 1: - self._bufsize = bufsize - self._flags |= self.FLAG_BUFFERED - self._flags &= ~self.FLAG_LINE_BUFFERED - elif bufsize == 0: - # unbuffered - self._flags &= ~(self.FLAG_BUFFERED | self.FLAG_LINE_BUFFERED) - - if ('r' in mode) or ('+' in mode): - self._flags |= self.FLAG_READ - if ('w' in mode) or ('+' in mode): - self._flags |= self.FLAG_WRITE - if ('a' in mode): - self._flags |= self.FLAG_WRITE | self.FLAG_APPEND - self._size = self._get_size() - self._pos = self._realpos = self._size - if ('b' in mode): - self._flags |= self.FLAG_BINARY - if ('U' in mode): - self._flags |= self.FLAG_UNIVERSAL_NEWLINE - # built-in file objects have this attribute to store which kinds of - # line terminations they've seen: - # - self.newlines = None - - def _write_all(self, data): - # the underlying stream may be something that does partial writes (like - # a socket). - while len(data) > 0: - count = self._write(data) - data = data[count:] - if self._flags & self.FLAG_APPEND: - self._size += count - self._pos = self._realpos = self._size - else: - self._pos += count - self._realpos += count - return None - - def _record_newline(self, newline): - # silliness about tracking what kinds of newlines we've seen. - # i don't understand why it can be None, a string, or a tuple, instead - # of just always being a tuple, but we'll emulate that behavior anyway. - if not (self._flags & self.FLAG_UNIVERSAL_NEWLINE): - return - if self.newlines is None: - self.newlines = newline - elif (type(self.newlines) is str) and (self.newlines != newline): - self.newlines = (self.newlines, newline) - elif newline not in self.newlines: - self.newlines += (newline,) diff --git a/contrib/site-packages/paramiko/hostkeys.py b/contrib/site-packages/paramiko/hostkeys.py deleted file mode 100644 index 9bcf0d55..00000000 --- a/contrib/site-packages/paramiko/hostkeys.py +++ /dev/null @@ -1,342 +0,0 @@ -# Copyright (C) 2006-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -L{HostKeys} -""" - -import base64 -import binascii -from Crypto.Hash import SHA, HMAC -import UserDict - -from paramiko.common import * -from paramiko.dsskey import DSSKey -from paramiko.rsakey import RSAKey -from paramiko.util import get_logger -from paramiko.ecdsakey import ECDSAKey - - -class InvalidHostKey(Exception): - - def __init__(self, line, exc): - self.line = line - self.exc = exc - self.args = (line, exc) - - -class HostKeyEntry: - """ - Representation of a line in an OpenSSH-style "known hosts" file. - """ - - def __init__(self, hostnames=None, key=None): - self.valid = (hostnames is not None) and (key is not None) - self.hostnames = hostnames - self.key = key - - def from_line(cls, line, lineno=None): - """ - Parses the given line of text to find the names for the host, - the type of key, and the key data. The line is expected to be in the - format used by the openssh known_hosts file. - - Lines are expected to not have leading or trailing whitespace. - We don't bother to check for comments or empty lines. All of - that should be taken care of before sending the line to us. - - @param line: a line from an OpenSSH known_hosts file - @type line: str - """ - log = get_logger('paramiko.hostkeys') - fields = line.split(' ') - if len(fields) < 3: - # Bad number of fields - log.info("Not enough fields found in known_hosts in line %s (%r)" % - (lineno, line)) - return None - fields = fields[:3] - - names, keytype, key = fields - names = names.split(',') - - # Decide what kind of key we're looking at and create an object - # to hold it accordingly. - try: - if keytype == 'ssh-rsa': - key = RSAKey(data=base64.decodestring(key)) - elif keytype == 'ssh-dss': - key = DSSKey(data=base64.decodestring(key)) - elif keytype == 'ecdsa-sha2-nistp256': - key = ECDSAKey(data=base64.decodestring(key)) - else: - log.info("Unable to handle key of type %s" % (keytype,)) - return None - - except binascii.Error, e: - raise InvalidHostKey(line, e) - - return cls(names, key) - from_line = classmethod(from_line) - - def to_line(self): - """ - Returns a string in OpenSSH known_hosts file format, or None if - the object is not in a valid state. A trailing newline is - included. - """ - if self.valid: - return '%s %s %s\n' % (','.join(self.hostnames), self.key.get_name(), - self.key.get_base64()) - return None - - def __repr__(self): - return '' % (self.hostnames, self.key) - - -class HostKeys (UserDict.DictMixin): - """ - Representation of an openssh-style "known hosts" file. Host keys can be - read from one or more files, and then individual hosts can be looked up to - verify server keys during SSH negotiation. - - A HostKeys object can be treated like a dict; any dict lookup is equivalent - to calling L{lookup}. - - @since: 1.5.3 - """ - - def __init__(self, filename=None): - """ - Create a new HostKeys object, optionally loading keys from an openssh - style host-key file. - - @param filename: filename to load host keys from, or C{None} - @type filename: str - """ - # emulate a dict of { hostname: { keytype: PKey } } - self._entries = [] - if filename is not None: - self.load(filename) - - def add(self, hostname, keytype, key): - """ - Add a host key entry to the table. Any existing entry for a - C{(hostname, keytype)} pair will be replaced. - - @param hostname: the hostname (or IP) to add - @type hostname: str - @param keytype: key type (C{"ssh-rsa"} or C{"ssh-dss"}) - @type keytype: str - @param key: the key to add - @type key: L{PKey} - """ - for e in self._entries: - if (hostname in e.hostnames) and (e.key.get_name() == keytype): - e.key = key - return - self._entries.append(HostKeyEntry([hostname], key)) - - def load(self, filename): - """ - Read a file of known SSH host keys, in the format used by openssh. - This type of file unfortunately doesn't exist on Windows, but on - posix, it will usually be stored in - C{os.path.expanduser("~/.ssh/known_hosts")}. - - If this method is called multiple times, the host keys are merged, - not cleared. So multiple calls to C{load} will just call L{add}, - replacing any existing entries and adding new ones. - - @param filename: name of the file to read host keys from - @type filename: str - - @raise IOError: if there was an error reading the file - """ - f = open(filename, 'r') - for lineno, line in enumerate(f): - line = line.strip() - if (len(line) == 0) or (line[0] == '#'): - continue - e = HostKeyEntry.from_line(line, lineno) - if e is not None: - _hostnames = e.hostnames - for h in _hostnames: - if self.check(h, e.key): - e.hostnames.remove(h) - if len(e.hostnames): - self._entries.append(e) - f.close() - - def save(self, filename): - """ - Save host keys into a file, in the format used by openssh. The order of - keys in the file will be preserved when possible (if these keys were - loaded from a file originally). The single exception is that combined - lines will be split into individual key lines, which is arguably a bug. - - @param filename: name of the file to write - @type filename: str - - @raise IOError: if there was an error writing the file - - @since: 1.6.1 - """ - f = open(filename, 'w') - for e in self._entries: - line = e.to_line() - if line: - f.write(line) - f.close() - - def lookup(self, hostname): - """ - Find a hostkey entry for a given hostname or IP. If no entry is found, - C{None} is returned. Otherwise a dictionary of keytype to key is - returned. The keytype will be either C{"ssh-rsa"} or C{"ssh-dss"}. - - @param hostname: the hostname (or IP) to lookup - @type hostname: str - @return: keys associated with this host (or C{None}) - @rtype: dict(str, L{PKey}) - """ - class SubDict (UserDict.DictMixin): - def __init__(self, hostname, entries, hostkeys): - self._hostname = hostname - self._entries = entries - self._hostkeys = hostkeys - - def __getitem__(self, key): - for e in self._entries: - if e.key.get_name() == key: - return e.key - raise KeyError(key) - - def __setitem__(self, key, val): - for e in self._entries: - if e.key is None: - continue - if e.key.get_name() == key: - # replace - e.key = val - break - else: - # add a new one - e = HostKeyEntry([hostname], val) - self._entries.append(e) - self._hostkeys._entries.append(e) - - def keys(self): - return [e.key.get_name() for e in self._entries if e.key is not None] - - entries = [] - for e in self._entries: - for h in e.hostnames: - if (h.startswith('|1|') and (self.hash_host(hostname, h) == h)) or (h == hostname): - entries.append(e) - if len(entries) == 0: - return None - return SubDict(hostname, entries, self) - - def check(self, hostname, key): - """ - Return True if the given key is associated with the given hostname - in this dictionary. - - @param hostname: hostname (or IP) of the SSH server - @type hostname: str - @param key: the key to check - @type key: L{PKey} - @return: C{True} if the key is associated with the hostname; C{False} - if not - @rtype: bool - """ - k = self.lookup(hostname) - if k is None: - return False - host_key = k.get(key.get_name(), None) - if host_key is None: - return False - return str(host_key) == str(key) - - def clear(self): - """ - Remove all host keys from the dictionary. - """ - self._entries = [] - - def __getitem__(self, key): - ret = self.lookup(key) - if ret is None: - raise KeyError(key) - return ret - - def __setitem__(self, hostname, entry): - # don't use this please. - if len(entry) == 0: - self._entries.append(HostKeyEntry([hostname], None)) - return - for key_type in entry.keys(): - found = False - for e in self._entries: - if (hostname in e.hostnames) and (e.key.get_name() == key_type): - # replace - e.key = entry[key_type] - found = True - if not found: - self._entries.append(HostKeyEntry([hostname], entry[key_type])) - - def keys(self): - # python 2.4 sets would be nice here. - ret = [] - for e in self._entries: - for h in e.hostnames: - if h not in ret: - ret.append(h) - return ret - - def values(self): - ret = [] - for k in self.keys(): - ret.append(self.lookup(k)) - return ret - - def hash_host(hostname, salt=None): - """ - Return a "hashed" form of the hostname, as used by openssh when storing - hashed hostnames in the known_hosts file. - - @param hostname: the hostname to hash - @type hostname: str - @param salt: optional salt to use when hashing (must be 20 bytes long) - @type salt: str - @return: the hashed hostname - @rtype: str - """ - if salt is None: - salt = rng.read(SHA.digest_size) - else: - if salt.startswith('|1|'): - salt = salt.split('|')[2] - salt = base64.decodestring(salt) - assert len(salt) == SHA.digest_size - hmac = HMAC.HMAC(salt, hostname, SHA).digest() - hostkey = '|1|%s|%s' % (base64.encodestring(salt), base64.encodestring(hmac)) - return hostkey.replace('\n', '') - hash_host = staticmethod(hash_host) - diff --git a/contrib/site-packages/paramiko/kex_gex.py b/contrib/site-packages/paramiko/kex_gex.py deleted file mode 100644 index c0455a1f..00000000 --- a/contrib/site-packages/paramiko/kex_gex.py +++ /dev/null @@ -1,243 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -Variant on L{KexGroup1 } where the prime "p" and -generator "g" are provided by the server. A bit more work is required on the -client side, and a B{lot} more on the server side. -""" - -from Crypto.Hash import SHA -from Crypto.Util import number - -from paramiko.common import * -from paramiko import util -from paramiko.message import Message -from paramiko.ssh_exception import SSHException - - -_MSG_KEXDH_GEX_REQUEST_OLD, _MSG_KEXDH_GEX_GROUP, _MSG_KEXDH_GEX_INIT, \ - _MSG_KEXDH_GEX_REPLY, _MSG_KEXDH_GEX_REQUEST = range(30, 35) - - -class KexGex (object): - - name = 'diffie-hellman-group-exchange-sha1' - min_bits = 1024 - max_bits = 8192 - preferred_bits = 2048 - - def __init__(self, transport): - self.transport = transport - self.p = None - self.q = None - self.g = None - self.x = None - self.e = None - self.f = None - self.old_style = False - - def start_kex(self, _test_old_style=False): - if self.transport.server_mode: - self.transport._expect_packet(_MSG_KEXDH_GEX_REQUEST, _MSG_KEXDH_GEX_REQUEST_OLD) - return - # request a bit range: we accept (min_bits) to (max_bits), but prefer - # (preferred_bits). according to the spec, we shouldn't pull the - # minimum up above 1024. - m = Message() - if _test_old_style: - # only used for unit tests: we shouldn't ever send this - m.add_byte(chr(_MSG_KEXDH_GEX_REQUEST_OLD)) - m.add_int(self.preferred_bits) - self.old_style = True - else: - m.add_byte(chr(_MSG_KEXDH_GEX_REQUEST)) - m.add_int(self.min_bits) - m.add_int(self.preferred_bits) - m.add_int(self.max_bits) - self.transport._send_message(m) - self.transport._expect_packet(_MSG_KEXDH_GEX_GROUP) - - def parse_next(self, ptype, m): - if ptype == _MSG_KEXDH_GEX_REQUEST: - return self._parse_kexdh_gex_request(m) - elif ptype == _MSG_KEXDH_GEX_GROUP: - return self._parse_kexdh_gex_group(m) - elif ptype == _MSG_KEXDH_GEX_INIT: - return self._parse_kexdh_gex_init(m) - elif ptype == _MSG_KEXDH_GEX_REPLY: - return self._parse_kexdh_gex_reply(m) - elif ptype == _MSG_KEXDH_GEX_REQUEST_OLD: - return self._parse_kexdh_gex_request_old(m) - raise SSHException('KexGex asked to handle packet type %d' % ptype) - - - ### internals... - - - def _generate_x(self): - # generate an "x" (1 < x < (p-1)/2). - q = (self.p - 1) // 2 - qnorm = util.deflate_long(q, 0) - qhbyte = ord(qnorm[0]) - bytes = len(qnorm) - qmask = 0xff - while not (qhbyte & 0x80): - qhbyte <<= 1 - qmask >>= 1 - while True: - x_bytes = self.transport.rng.read(bytes) - x_bytes = chr(ord(x_bytes[0]) & qmask) + x_bytes[1:] - x = util.inflate_long(x_bytes, 1) - if (x > 1) and (x < q): - break - self.x = x - - def _parse_kexdh_gex_request(self, m): - minbits = m.get_int() - preferredbits = m.get_int() - maxbits = m.get_int() - # smoosh the user's preferred size into our own limits - if preferredbits > self.max_bits: - preferredbits = self.max_bits - if preferredbits < self.min_bits: - preferredbits = self.min_bits - # fix min/max if they're inconsistent. technically, we could just pout - # and hang up, but there's no harm in giving them the benefit of the - # doubt and just picking a bitsize for them. - if minbits > preferredbits: - minbits = preferredbits - if maxbits < preferredbits: - maxbits = preferredbits - # now save a copy - self.min_bits = minbits - self.preferred_bits = preferredbits - self.max_bits = maxbits - # generate prime - pack = self.transport._get_modulus_pack() - if pack is None: - raise SSHException('Can\'t do server-side gex with no modulus pack') - self.transport._log(DEBUG, 'Picking p (%d <= %d <= %d bits)' % (minbits, preferredbits, maxbits)) - self.g, self.p = pack.get_modulus(minbits, preferredbits, maxbits) - m = Message() - m.add_byte(chr(_MSG_KEXDH_GEX_GROUP)) - m.add_mpint(self.p) - m.add_mpint(self.g) - self.transport._send_message(m) - self.transport._expect_packet(_MSG_KEXDH_GEX_INIT) - - def _parse_kexdh_gex_request_old(self, m): - # same as above, but without min_bits or max_bits (used by older clients like putty) - self.preferred_bits = m.get_int() - # smoosh the user's preferred size into our own limits - if self.preferred_bits > self.max_bits: - self.preferred_bits = self.max_bits - if self.preferred_bits < self.min_bits: - self.preferred_bits = self.min_bits - # generate prime - pack = self.transport._get_modulus_pack() - if pack is None: - raise SSHException('Can\'t do server-side gex with no modulus pack') - self.transport._log(DEBUG, 'Picking p (~ %d bits)' % (self.preferred_bits,)) - self.g, self.p = pack.get_modulus(self.min_bits, self.preferred_bits, self.max_bits) - m = Message() - m.add_byte(chr(_MSG_KEXDH_GEX_GROUP)) - m.add_mpint(self.p) - m.add_mpint(self.g) - self.transport._send_message(m) - self.transport._expect_packet(_MSG_KEXDH_GEX_INIT) - self.old_style = True - - def _parse_kexdh_gex_group(self, m): - self.p = m.get_mpint() - self.g = m.get_mpint() - # reject if p's bit length < 1024 or > 8192 - bitlen = util.bit_length(self.p) - if (bitlen < 1024) or (bitlen > 8192): - raise SSHException('Server-generated gex p (don\'t ask) is out of range (%d bits)' % bitlen) - self.transport._log(DEBUG, 'Got server p (%d bits)' % bitlen) - self._generate_x() - # now compute e = g^x mod p - self.e = pow(self.g, self.x, self.p) - m = Message() - m.add_byte(chr(_MSG_KEXDH_GEX_INIT)) - m.add_mpint(self.e) - self.transport._send_message(m) - self.transport._expect_packet(_MSG_KEXDH_GEX_REPLY) - - def _parse_kexdh_gex_init(self, m): - self.e = m.get_mpint() - if (self.e < 1) or (self.e > self.p - 1): - raise SSHException('Client kex "e" is out of range') - self._generate_x() - self.f = pow(self.g, self.x, self.p) - K = pow(self.e, self.x, self.p) - key = str(self.transport.get_server_key()) - # okay, build up the hash H of (V_C || V_S || I_C || I_S || K_S || min || n || max || p || g || e || f || K) - hm = Message() - hm.add(self.transport.remote_version, self.transport.local_version, - self.transport.remote_kex_init, self.transport.local_kex_init, - key) - if not self.old_style: - hm.add_int(self.min_bits) - hm.add_int(self.preferred_bits) - if not self.old_style: - hm.add_int(self.max_bits) - hm.add_mpint(self.p) - hm.add_mpint(self.g) - hm.add_mpint(self.e) - hm.add_mpint(self.f) - hm.add_mpint(K) - H = SHA.new(str(hm)).digest() - self.transport._set_K_H(K, H) - # sign it - sig = self.transport.get_server_key().sign_ssh_data(self.transport.rng, H) - # send reply - m = Message() - m.add_byte(chr(_MSG_KEXDH_GEX_REPLY)) - m.add_string(key) - m.add_mpint(self.f) - m.add_string(str(sig)) - self.transport._send_message(m) - self.transport._activate_outbound() - - def _parse_kexdh_gex_reply(self, m): - host_key = m.get_string() - self.f = m.get_mpint() - sig = m.get_string() - if (self.f < 1) or (self.f > self.p - 1): - raise SSHException('Server kex "f" is out of range') - K = pow(self.f, self.x, self.p) - # okay, build up the hash H of (V_C || V_S || I_C || I_S || K_S || min || n || max || p || g || e || f || K) - hm = Message() - hm.add(self.transport.local_version, self.transport.remote_version, - self.transport.local_kex_init, self.transport.remote_kex_init, - host_key) - if not self.old_style: - hm.add_int(self.min_bits) - hm.add_int(self.preferred_bits) - if not self.old_style: - hm.add_int(self.max_bits) - hm.add_mpint(self.p) - hm.add_mpint(self.g) - hm.add_mpint(self.e) - hm.add_mpint(self.f) - hm.add_mpint(K) - self.transport._set_K_H(K, SHA.new(str(hm)).digest()) - self.transport._verify_key(host_key, sig) - self.transport._activate_outbound() diff --git a/contrib/site-packages/paramiko/kex_group1.py b/contrib/site-packages/paramiko/kex_group1.py deleted file mode 100644 index 6e89b6dc..00000000 --- a/contrib/site-packages/paramiko/kex_group1.py +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -Standard SSH key exchange ("kex" if you wanna sound cool). Diffie-Hellman of -1024 bit key halves, using a known "p" prime and "g" generator. -""" - -from Crypto.Hash import SHA - -from paramiko.common import * -from paramiko import util -from paramiko.message import Message -from paramiko.ssh_exception import SSHException - - -_MSG_KEXDH_INIT, _MSG_KEXDH_REPLY = range(30, 32) - -# draft-ietf-secsh-transport-09.txt, page 17 -P = 0xFFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E088A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE649286651ECE65381FFFFFFFFFFFFFFFFL -G = 2 - - -class KexGroup1(object): - - name = 'diffie-hellman-group1-sha1' - - def __init__(self, transport): - self.transport = transport - self.x = 0L - self.e = 0L - self.f = 0L - - def start_kex(self): - self._generate_x() - if self.transport.server_mode: - # compute f = g^x mod p, but don't send it yet - self.f = pow(G, self.x, P) - self.transport._expect_packet(_MSG_KEXDH_INIT) - return - # compute e = g^x mod p (where g=2), and send it - self.e = pow(G, self.x, P) - m = Message() - m.add_byte(chr(_MSG_KEXDH_INIT)) - m.add_mpint(self.e) - self.transport._send_message(m) - self.transport._expect_packet(_MSG_KEXDH_REPLY) - - def parse_next(self, ptype, m): - if self.transport.server_mode and (ptype == _MSG_KEXDH_INIT): - return self._parse_kexdh_init(m) - elif not self.transport.server_mode and (ptype == _MSG_KEXDH_REPLY): - return self._parse_kexdh_reply(m) - raise SSHException('KexGroup1 asked to handle packet type %d' % ptype) - - - ### internals... - - - def _generate_x(self): - # generate an "x" (1 < x < q), where q is (p-1)/2. - # p is a 128-byte (1024-bit) number, where the first 64 bits are 1. - # therefore q can be approximated as a 2^1023. we drop the subset of - # potential x where the first 63 bits are 1, because some of those will be - # larger than q (but this is a tiny tiny subset of potential x). - while 1: - x_bytes = self.transport.rng.read(128) - x_bytes = chr(ord(x_bytes[0]) & 0x7f) + x_bytes[1:] - if (x_bytes[:8] != '\x7F\xFF\xFF\xFF\xFF\xFF\xFF\xFF') and \ - (x_bytes[:8] != '\x00\x00\x00\x00\x00\x00\x00\x00'): - break - self.x = util.inflate_long(x_bytes) - - def _parse_kexdh_reply(self, m): - # client mode - host_key = m.get_string() - self.f = m.get_mpint() - if (self.f < 1) or (self.f > P - 1): - raise SSHException('Server kex "f" is out of range') - sig = m.get_string() - K = pow(self.f, self.x, P) - # okay, build up the hash H of (V_C || V_S || I_C || I_S || K_S || e || f || K) - hm = Message() - hm.add(self.transport.local_version, self.transport.remote_version, - self.transport.local_kex_init, self.transport.remote_kex_init) - hm.add_string(host_key) - hm.add_mpint(self.e) - hm.add_mpint(self.f) - hm.add_mpint(K) - self.transport._set_K_H(K, SHA.new(str(hm)).digest()) - self.transport._verify_key(host_key, sig) - self.transport._activate_outbound() - - def _parse_kexdh_init(self, m): - # server mode - self.e = m.get_mpint() - if (self.e < 1) or (self.e > P - 1): - raise SSHException('Client kex "e" is out of range') - K = pow(self.e, self.x, P) - key = str(self.transport.get_server_key()) - # okay, build up the hash H of (V_C || V_S || I_C || I_S || K_S || e || f || K) - hm = Message() - hm.add(self.transport.remote_version, self.transport.local_version, - self.transport.remote_kex_init, self.transport.local_kex_init) - hm.add_string(key) - hm.add_mpint(self.e) - hm.add_mpint(self.f) - hm.add_mpint(K) - H = SHA.new(str(hm)).digest() - self.transport._set_K_H(K, H) - # sign it - sig = self.transport.get_server_key().sign_ssh_data(self.transport.rng, H) - # send reply - m = Message() - m.add_byte(chr(_MSG_KEXDH_REPLY)) - m.add_string(key) - m.add_mpint(self.f) - m.add_string(str(sig)) - self.transport._send_message(m) - self.transport._activate_outbound() diff --git a/contrib/site-packages/paramiko/logging22.py b/contrib/site-packages/paramiko/logging22.py deleted file mode 100644 index e68c52cb..00000000 --- a/contrib/site-packages/paramiko/logging22.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -Stub out logging on python < 2.3. -""" - - -DEBUG = 10 -INFO = 20 -WARNING = 30 -ERROR = 40 -CRITICAL = 50 - - -def getLogger(name): - return _logger - - -class logger (object): - def __init__(self): - self.handlers = [ ] - self.level = ERROR - - def setLevel(self, level): - self.level = level - - def addHandler(self, h): - self.handlers.append(h) - - def addFilter(self, filter): - pass - - def log(self, level, text): - if level >= self.level: - for h in self.handlers: - h.f.write(text + '\n') - h.f.flush() - -class StreamHandler (object): - def __init__(self, f): - self.f = f - - def setFormatter(self, f): - pass - -class Formatter (object): - def __init__(self, x, y): - pass - -_logger = logger() diff --git a/contrib/site-packages/paramiko/message.py b/contrib/site-packages/paramiko/message.py deleted file mode 100644 index c0e8692b..00000000 --- a/contrib/site-packages/paramiko/message.py +++ /dev/null @@ -1,302 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -Implementation of an SSH2 "message". -""" - -import struct -import cStringIO - -from paramiko import util - - -class Message (object): - """ - An SSH2 I{Message} is a stream of bytes that encodes some combination of - strings, integers, bools, and infinite-precision integers (known in python - as I{long}s). This class builds or breaks down such a byte stream. - - Normally you don't need to deal with anything this low-level, but it's - exposed for people implementing custom extensions, or features that - paramiko doesn't support yet. - """ - - def __init__(self, content=None): - """ - Create a new SSH2 Message. - - @param content: the byte stream to use as the Message content (passed - in only when decomposing a Message). - @type content: string - """ - if content != None: - self.packet = cStringIO.StringIO(content) - else: - self.packet = cStringIO.StringIO() - - def __str__(self): - """ - Return the byte stream content of this Message, as a string. - - @return: the contents of this Message. - @rtype: string - """ - return self.packet.getvalue() - - def __repr__(self): - """ - Returns a string representation of this object, for debugging. - - @rtype: string - """ - return 'paramiko.Message(' + repr(self.packet.getvalue()) + ')' - - def rewind(self): - """ - Rewind the message to the beginning as if no items had been parsed - out of it yet. - """ - self.packet.seek(0) - - def get_remainder(self): - """ - Return the bytes of this Message that haven't already been parsed and - returned. - - @return: a string of the bytes not parsed yet. - @rtype: string - """ - position = self.packet.tell() - remainder = self.packet.read() - self.packet.seek(position) - return remainder - - def get_so_far(self): - """ - Returns the bytes of this Message that have been parsed and returned. - The string passed into a Message's constructor can be regenerated by - concatenating C{get_so_far} and L{get_remainder}. - - @return: a string of the bytes parsed so far. - @rtype: string - """ - position = self.packet.tell() - self.rewind() - return self.packet.read(position) - - def get_bytes(self, n): - """ - Return the next C{n} bytes of the Message, without decomposing into - an int, string, etc. Just the raw bytes are returned. - - @return: a string of the next C{n} bytes of the Message, or a string - of C{n} zero bytes, if there aren't C{n} bytes remaining. - @rtype: string - """ - b = self.packet.read(n) - max_pad_size = 1<<20 # Limit padding to 1 MB - if len(b) < n and n < max_pad_size: - return b + '\x00' * (n - len(b)) - return b - - def get_byte(self): - """ - Return the next byte of the Message, without decomposing it. This - is equivalent to L{get_bytes(1)}. - - @return: the next byte of the Message, or C{'\000'} if there aren't - any bytes remaining. - @rtype: string - """ - return self.get_bytes(1) - - def get_boolean(self): - """ - Fetch a boolean from the stream. - - @return: C{True} or C{False} (from the Message). - @rtype: bool - """ - b = self.get_bytes(1) - return b != '\x00' - - def get_int(self): - """ - Fetch an int from the stream. - - @return: a 32-bit unsigned integer. - @rtype: int - """ - return struct.unpack('>I', self.get_bytes(4))[0] - - def get_int64(self): - """ - Fetch a 64-bit int from the stream. - - @return: a 64-bit unsigned integer. - @rtype: long - """ - return struct.unpack('>Q', self.get_bytes(8))[0] - - def get_mpint(self): - """ - Fetch a long int (mpint) from the stream. - - @return: an arbitrary-length integer. - @rtype: long - """ - return util.inflate_long(self.get_string()) - - def get_string(self): - """ - Fetch a string from the stream. This could be a byte string and may - contain unprintable characters. (It's not unheard of for a string to - contain another byte-stream Message.) - - @return: a string. - @rtype: string - """ - return self.get_bytes(self.get_int()) - - def get_list(self): - """ - Fetch a list of strings from the stream. These are trivially encoded - as comma-separated values in a string. - - @return: a list of strings. - @rtype: list of strings - """ - return self.get_string().split(',') - - def add_bytes(self, b): - """ - Write bytes to the stream, without any formatting. - - @param b: bytes to add - @type b: str - """ - self.packet.write(b) - return self - - def add_byte(self, b): - """ - Write a single byte to the stream, without any formatting. - - @param b: byte to add - @type b: str - """ - self.packet.write(b) - return self - - def add_boolean(self, b): - """ - Add a boolean value to the stream. - - @param b: boolean value to add - @type b: bool - """ - if b: - self.add_byte('\x01') - else: - self.add_byte('\x00') - return self - - def add_int(self, n): - """ - Add an integer to the stream. - - @param n: integer to add - @type n: int - """ - self.packet.write(struct.pack('>I', n)) - return self - - def add_int64(self, n): - """ - Add a 64-bit int to the stream. - - @param n: long int to add - @type n: long - """ - self.packet.write(struct.pack('>Q', n)) - return self - - def add_mpint(self, z): - """ - Add a long int to the stream, encoded as an infinite-precision - integer. This method only works on positive numbers. - - @param z: long int to add - @type z: long - """ - self.add_string(util.deflate_long(z)) - return self - - def add_string(self, s): - """ - Add a string to the stream. - - @param s: string to add - @type s: str - """ - self.add_int(len(s)) - self.packet.write(s) - return self - - def add_list(self, l): - """ - Add a list of strings to the stream. They are encoded identically to - a single string of values separated by commas. (Yes, really, that's - how SSH2 does it.) - - @param l: list of strings to add - @type l: list(str) - """ - self.add_string(','.join(l)) - return self - - def _add(self, i): - if type(i) is str: - return self.add_string(i) - elif type(i) is int: - return self.add_int(i) - elif type(i) is long: - if i > 0xffffffffL: - return self.add_mpint(i) - else: - return self.add_int(i) - elif type(i) is bool: - return self.add_boolean(i) - elif type(i) is list: - return self.add_list(i) - else: - raise Exception('Unknown type') - - def add(self, *seq): - """ - Add a sequence of items to the stream. The values are encoded based - on their type: str, int, bool, list, or long. - - @param seq: the sequence of items - @type seq: sequence - - @bug: longs are encoded non-deterministically. Don't use this method. - """ - for item in seq: - self._add(item) diff --git a/contrib/site-packages/paramiko/packet.py b/contrib/site-packages/paramiko/packet.py deleted file mode 100644 index 6ab7363d..00000000 --- a/contrib/site-packages/paramiko/packet.py +++ /dev/null @@ -1,500 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -Packetizer. -""" - -import errno -import select -import socket -import struct -import threading -import time - -from paramiko.common import * -from paramiko import util -from paramiko.ssh_exception import SSHException, ProxyCommandFailure -from paramiko.message import Message - - -try: - from r_hmac import HMAC -except ImportError: - from Crypto.Hash.HMAC import HMAC - -def compute_hmac(key, message, digest_class): - return HMAC(key, message, digest_class).digest() - - -class NeedRekeyException (Exception): - pass - - -class Packetizer (object): - """ - Implementation of the base SSH packet protocol. - """ - - # READ the secsh RFC's before raising these values. if anything, - # they should probably be lower. - REKEY_PACKETS = pow(2, 29) - REKEY_BYTES = pow(2, 29) - - REKEY_PACKETS_OVERFLOW_MAX = pow(2,29) # Allow receiving this many packets after a re-key request before terminating - REKEY_BYTES_OVERFLOW_MAX = pow(2,29) # Allow receiving this many bytes after a re-key request before terminating - - def __init__(self, socket): - self.__socket = socket - self.__logger = None - self.__closed = False - self.__dump_packets = False - self.__need_rekey = False - self.__init_count = 0 - self.__remainder = '' - - # used for noticing when to re-key: - self.__sent_bytes = 0 - self.__sent_packets = 0 - self.__received_bytes = 0 - self.__received_packets = 0 - self.__received_bytes_overflow = 0 - self.__received_packets_overflow = 0 - - # current inbound/outbound ciphering: - self.__block_size_out = 8 - self.__block_size_in = 8 - self.__mac_size_out = 0 - self.__mac_size_in = 0 - self.__block_engine_out = None - self.__block_engine_in = None - self.__sdctr_out = False - self.__mac_engine_out = None - self.__mac_engine_in = None - self.__mac_key_out = '' - self.__mac_key_in = '' - self.__compress_engine_out = None - self.__compress_engine_in = None - self.__sequence_number_out = 0L - self.__sequence_number_in = 0L - - # lock around outbound writes (packet computation) - self.__write_lock = threading.RLock() - - # keepalives: - self.__keepalive_interval = 0 - self.__keepalive_last = time.time() - self.__keepalive_callback = None - - def set_log(self, log): - """ - Set the python log object to use for logging. - """ - self.__logger = log - - def set_outbound_cipher(self, block_engine, block_size, mac_engine, mac_size, mac_key, sdctr=False): - """ - Switch outbound data cipher. - """ - self.__block_engine_out = block_engine - self.__sdctr_out = sdctr - self.__block_size_out = block_size - self.__mac_engine_out = mac_engine - self.__mac_size_out = mac_size - self.__mac_key_out = mac_key - self.__sent_bytes = 0 - self.__sent_packets = 0 - # wait until the reset happens in both directions before clearing rekey flag - self.__init_count |= 1 - if self.__init_count == 3: - self.__init_count = 0 - self.__need_rekey = False - - def set_inbound_cipher(self, block_engine, block_size, mac_engine, mac_size, mac_key): - """ - Switch inbound data cipher. - """ - self.__block_engine_in = block_engine - self.__block_size_in = block_size - self.__mac_engine_in = mac_engine - self.__mac_size_in = mac_size - self.__mac_key_in = mac_key - self.__received_bytes = 0 - self.__received_packets = 0 - self.__received_bytes_overflow = 0 - self.__received_packets_overflow = 0 - # wait until the reset happens in both directions before clearing rekey flag - self.__init_count |= 2 - if self.__init_count == 3: - self.__init_count = 0 - self.__need_rekey = False - - def set_outbound_compressor(self, compressor): - self.__compress_engine_out = compressor - - def set_inbound_compressor(self, compressor): - self.__compress_engine_in = compressor - - def close(self): - self.__closed = True - - def set_hexdump(self, hexdump): - self.__dump_packets = hexdump - - def get_hexdump(self): - return self.__dump_packets - - def get_mac_size_in(self): - return self.__mac_size_in - - def get_mac_size_out(self): - return self.__mac_size_out - - def need_rekey(self): - """ - Returns C{True} if a new set of keys needs to be negotiated. This - will be triggered during a packet read or write, so it should be - checked after every read or write, or at least after every few. - - @return: C{True} if a new set of keys needs to be negotiated - """ - return self.__need_rekey - - def set_keepalive(self, interval, callback): - """ - Turn on/off the callback keepalive. If C{interval} seconds pass with - no data read from or written to the socket, the callback will be - executed and the timer will be reset. - """ - self.__keepalive_interval = interval - self.__keepalive_callback = callback - self.__keepalive_last = time.time() - - def read_all(self, n, check_rekey=False): - """ - Read as close to N bytes as possible, blocking as long as necessary. - - @param n: number of bytes to read - @type n: int - @return: the data read - @rtype: str - @raise EOFError: if the socket was closed before all the bytes could - be read - """ - out = '' - # handle over-reading from reading the banner line - if len(self.__remainder) > 0: - out = self.__remainder[:n] - self.__remainder = self.__remainder[n:] - n -= len(out) - if PY22: - return self._py22_read_all(n, out) - while n > 0: - got_timeout = False - try: - x = self.__socket.recv(n) - if len(x) == 0: - raise EOFError() - out += x - n -= len(x) - except socket.timeout: - got_timeout = True - except socket.error, e: - # on Linux, sometimes instead of socket.timeout, we get - # EAGAIN. this is a bug in recent (> 2.6.9) kernels but - # we need to work around it. - if (type(e.args) is tuple) and (len(e.args) > 0) and (e.args[0] == errno.EAGAIN): - got_timeout = True - elif (type(e.args) is tuple) and (len(e.args) > 0) and (e.args[0] == errno.EINTR): - # syscall interrupted; try again - pass - elif self.__closed: - raise EOFError() - else: - raise - if got_timeout: - if self.__closed: - raise EOFError() - if check_rekey and (len(out) == 0) and self.__need_rekey: - raise NeedRekeyException() - self._check_keepalive() - return out - - def write_all(self, out): - self.__keepalive_last = time.time() - while len(out) > 0: - retry_write = False - try: - n = self.__socket.send(out) - except socket.timeout: - retry_write = True - except socket.error, e: - if (type(e.args) is tuple) and (len(e.args) > 0) and (e.args[0] == errno.EAGAIN): - retry_write = True - elif (type(e.args) is tuple) and (len(e.args) > 0) and (e.args[0] == errno.EINTR): - # syscall interrupted; try again - retry_write = True - else: - n = -1 - except ProxyCommandFailure: - raise # so it doesn't get swallowed by the below catchall - except Exception: - # could be: (32, 'Broken pipe') - n = -1 - if retry_write: - n = 0 - if self.__closed: - n = -1 - if n < 0: - raise EOFError() - if n == len(out): - break - out = out[n:] - return - - def readline(self, timeout): - """ - Read a line from the socket. We assume no data is pending after the - line, so it's okay to attempt large reads. - """ - buf = self.__remainder - while not '\n' in buf: - buf += self._read_timeout(timeout) - n = buf.index('\n') - self.__remainder = buf[n+1:] - buf = buf[:n] - if (len(buf) > 0) and (buf[-1] == '\r'): - buf = buf[:-1] - return buf - - def send_message(self, data): - """ - Write a block of data using the current cipher, as an SSH block. - """ - # encrypt this sucka - data = str(data) - cmd = ord(data[0]) - if cmd in MSG_NAMES: - cmd_name = MSG_NAMES[cmd] - else: - cmd_name = '$%x' % cmd - orig_len = len(data) - self.__write_lock.acquire() - try: - if self.__compress_engine_out is not None: - data = self.__compress_engine_out(data) - packet = self._build_packet(data) - if self.__dump_packets: - self._log(DEBUG, 'Write packet <%s>, length %d' % (cmd_name, orig_len)) - self._log(DEBUG, util.format_binary(packet, 'OUT: ')) - if self.__block_engine_out != None: - out = self.__block_engine_out.encrypt(packet) - else: - out = packet - # + mac - if self.__block_engine_out != None: - payload = struct.pack('>I', self.__sequence_number_out) + packet - out += compute_hmac(self.__mac_key_out, payload, self.__mac_engine_out)[:self.__mac_size_out] - self.__sequence_number_out = (self.__sequence_number_out + 1) & 0xffffffffL - self.write_all(out) - - self.__sent_bytes += len(out) - self.__sent_packets += 1 - if ((self.__sent_packets >= self.REKEY_PACKETS) or (self.__sent_bytes >= self.REKEY_BYTES)) \ - and not self.__need_rekey: - # only ask once for rekeying - self._log(DEBUG, 'Rekeying (hit %d packets, %d bytes sent)' % - (self.__sent_packets, self.__sent_bytes)) - self.__received_bytes_overflow = 0 - self.__received_packets_overflow = 0 - self._trigger_rekey() - finally: - self.__write_lock.release() - - def read_message(self): - """ - Only one thread should ever be in this function (no other locking is - done). - - @raise SSHException: if the packet is mangled - @raise NeedRekeyException: if the transport should rekey - """ - header = self.read_all(self.__block_size_in, check_rekey=True) - if self.__block_engine_in != None: - header = self.__block_engine_in.decrypt(header) - if self.__dump_packets: - self._log(DEBUG, util.format_binary(header, 'IN: ')); - packet_size = struct.unpack('>I', header[:4])[0] - # leftover contains decrypted bytes from the first block (after the length field) - leftover = header[4:] - if (packet_size - len(leftover)) % self.__block_size_in != 0: - raise SSHException('Invalid packet blocking') - buf = self.read_all(packet_size + self.__mac_size_in - len(leftover)) - packet = buf[:packet_size - len(leftover)] - post_packet = buf[packet_size - len(leftover):] - if self.__block_engine_in != None: - packet = self.__block_engine_in.decrypt(packet) - if self.__dump_packets: - self._log(DEBUG, util.format_binary(packet, 'IN: ')); - packet = leftover + packet - - if self.__mac_size_in > 0: - mac = post_packet[:self.__mac_size_in] - mac_payload = struct.pack('>II', self.__sequence_number_in, packet_size) + packet - my_mac = compute_hmac(self.__mac_key_in, mac_payload, self.__mac_engine_in)[:self.__mac_size_in] - if my_mac != mac: - raise SSHException('Mismatched MAC') - padding = ord(packet[0]) - payload = packet[1:packet_size - padding] - - if self.__dump_packets: - self._log(DEBUG, 'Got payload (%d bytes, %d padding)' % (packet_size, padding)) - - if self.__compress_engine_in is not None: - payload = self.__compress_engine_in(payload) - - msg = Message(payload[1:]) - msg.seqno = self.__sequence_number_in - self.__sequence_number_in = (self.__sequence_number_in + 1) & 0xffffffffL - - # check for rekey - raw_packet_size = packet_size + self.__mac_size_in + 4 - self.__received_bytes += raw_packet_size - self.__received_packets += 1 - if self.__need_rekey: - # we've asked to rekey -- give them some packets to comply before - # dropping the connection - self.__received_bytes_overflow += raw_packet_size - self.__received_packets_overflow += 1 - if (self.__received_packets_overflow >= self.REKEY_PACKETS_OVERFLOW_MAX) or \ - (self.__received_bytes_overflow >= self.REKEY_BYTES_OVERFLOW_MAX): - raise SSHException('Remote transport is ignoring rekey requests') - elif (self.__received_packets >= self.REKEY_PACKETS) or \ - (self.__received_bytes >= self.REKEY_BYTES): - # only ask once for rekeying - self._log(DEBUG, 'Rekeying (hit %d packets, %d bytes received)' % - (self.__received_packets, self.__received_bytes)) - self.__received_bytes_overflow = 0 - self.__received_packets_overflow = 0 - self._trigger_rekey() - - cmd = ord(payload[0]) - if cmd in MSG_NAMES: - cmd_name = MSG_NAMES[cmd] - else: - cmd_name = '$%x' % cmd - if self.__dump_packets: - self._log(DEBUG, 'Read packet <%s>, length %d' % (cmd_name, len(payload))) - return cmd, msg - - - ########## protected - - - def _log(self, level, msg): - if self.__logger is None: - return - if issubclass(type(msg), list): - for m in msg: - self.__logger.log(level, m) - else: - self.__logger.log(level, msg) - - def _check_keepalive(self): - if (not self.__keepalive_interval) or (not self.__block_engine_out) or \ - self.__need_rekey: - # wait till we're encrypting, and not in the middle of rekeying - return - now = time.time() - if now > self.__keepalive_last + self.__keepalive_interval: - self.__keepalive_callback() - self.__keepalive_last = now - - def _py22_read_all(self, n, out): - while n > 0: - r, w, e = select.select([self.__socket], [], [], 0.1) - if self.__socket not in r: - if self.__closed: - raise EOFError() - self._check_keepalive() - else: - x = self.__socket.recv(n) - if len(x) == 0: - raise EOFError() - out += x - n -= len(x) - return out - - def _py22_read_timeout(self, timeout): - start = time.time() - while True: - r, w, e = select.select([self.__socket], [], [], 0.1) - if self.__socket in r: - x = self.__socket.recv(1) - if len(x) == 0: - raise EOFError() - break - if self.__closed: - raise EOFError() - now = time.time() - if now - start >= timeout: - raise socket.timeout() - return x - - def _read_timeout(self, timeout): - if PY22: - return self._py22_read_timeout(timeout) - start = time.time() - while True: - try: - x = self.__socket.recv(128) - if len(x) == 0: - raise EOFError() - break - except socket.timeout: - pass - except EnvironmentError, e: - if ((type(e.args) is tuple) and (len(e.args) > 0) and - (e.args[0] == errno.EINTR)): - pass - else: - raise - if self.__closed: - raise EOFError() - now = time.time() - if now - start >= timeout: - raise socket.timeout() - return x - - def _build_packet(self, payload): - # pad up at least 4 bytes, to nearest block-size (usually 8) - bsize = self.__block_size_out - padding = 3 + bsize - ((len(payload) + 8) % bsize) - packet = struct.pack('>IB', len(payload) + padding + 1, padding) - packet += payload - if self.__sdctr_out or self.__block_engine_out is None: - # cute trick i caught openssh doing: if we're not encrypting or SDCTR mode (RFC4344), - # don't waste random bytes for the padding - packet += (chr(0) * padding) - else: - packet += rng.read(padding) - return packet - - def _trigger_rekey(self): - # outside code should check for this flag - self.__need_rekey = True diff --git a/contrib/site-packages/paramiko/pipe.py b/contrib/site-packages/paramiko/pipe.py deleted file mode 100644 index db43d549..00000000 --- a/contrib/site-packages/paramiko/pipe.py +++ /dev/null @@ -1,147 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -Abstraction of a one-way pipe where the read end can be used in select(). -Normally this is trivial, but Windows makes it nearly impossible. - -The pipe acts like an Event, which can be set or cleared. When set, the pipe -will trigger as readable in select(). -""" - -import sys -import os -import socket - - -def make_pipe (): - if sys.platform[:3] != 'win': - p = PosixPipe() - else: - p = WindowsPipe() - return p - - -class PosixPipe (object): - def __init__ (self): - self._rfd, self._wfd = os.pipe() - self._set = False - self._forever = False - self._closed = False - - def close (self): - os.close(self._rfd) - os.close(self._wfd) - # used for unit tests: - self._closed = True - - def fileno (self): - return self._rfd - - def clear (self): - if not self._set or self._forever: - return - os.read(self._rfd, 1) - self._set = False - - def set (self): - if self._set or self._closed: - return - self._set = True - os.write(self._wfd, '*') - - def set_forever (self): - self._forever = True - self.set() - - -class WindowsPipe (object): - """ - On Windows, only an OS-level "WinSock" may be used in select(), but reads - and writes must be to the actual socket object. - """ - def __init__ (self): - serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - serv.bind(('127.0.0.1', 0)) - serv.listen(1) - - # need to save sockets in _rsock/_wsock so they don't get closed - self._rsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self._rsock.connect(('127.0.0.1', serv.getsockname()[1])) - - self._wsock, addr = serv.accept() - serv.close() - self._set = False - self._forever = False - self._closed = False - - def close (self): - self._rsock.close() - self._wsock.close() - # used for unit tests: - self._closed = True - - def fileno (self): - return self._rsock.fileno() - - def clear (self): - if not self._set or self._forever: - return - self._rsock.recv(1) - self._set = False - - def set (self): - if self._set or self._closed: - return - self._set = True - self._wsock.send('*') - - def set_forever (self): - self._forever = True - self.set() - - -class OrPipe (object): - def __init__(self, pipe): - self._set = False - self._partner = None - self._pipe = pipe - - def set(self): - self._set = True - if not self._partner._set: - self._pipe.set() - - def clear(self): - self._set = False - if not self._partner._set: - self._pipe.clear() - - -def make_or_pipe(pipe): - """ - wraps a pipe into two pipe-like objects which are "or"d together to - affect the real pipe. if either returned pipe is set, the wrapped pipe - is set. when both are cleared, the wrapped pipe is cleared. - """ - p1 = OrPipe(pipe) - p2 = OrPipe(pipe) - p1._partner = p2 - p2._partner = p1 - return p1, p2 - diff --git a/contrib/site-packages/paramiko/pkey.py b/contrib/site-packages/paramiko/pkey.py deleted file mode 100644 index b1199df8..00000000 --- a/contrib/site-packages/paramiko/pkey.py +++ /dev/null @@ -1,381 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -Common API for all public keys. -""" - -import base64 -from binascii import hexlify, unhexlify -import os - -from Crypto.Hash import MD5 -from Crypto.Cipher import DES3, AES - -from paramiko.common import * -from paramiko import util -from paramiko.message import Message -from paramiko.ssh_exception import SSHException, PasswordRequiredException - - -class PKey (object): - """ - Base class for public keys. - """ - - # known encryption types for private key files: - _CIPHER_TABLE = { - 'AES-128-CBC': { 'cipher': AES, 'keysize': 16, 'blocksize': 16, 'mode': AES.MODE_CBC }, - 'DES-EDE3-CBC': { 'cipher': DES3, 'keysize': 24, 'blocksize': 8, 'mode': DES3.MODE_CBC }, - } - - - def __init__(self, msg=None, data=None): - """ - Create a new instance of this public key type. If C{msg} is given, - the key's public part(s) will be filled in from the message. If - C{data} is given, the key's public part(s) will be filled in from - the string. - - @param msg: an optional SSH L{Message} containing a public key of this - type. - @type msg: L{Message} - @param data: an optional string containing a public key of this type - @type data: str - - @raise SSHException: if a key cannot be created from the C{data} or - C{msg} given, or no key was passed in. - """ - pass - - def __str__(self): - """ - Return a string of an SSH L{Message} made up of the public part(s) of - this key. This string is suitable for passing to L{__init__} to - re-create the key object later. - - @return: string representation of an SSH key message. - @rtype: str - """ - return '' - - def __cmp__(self, other): - """ - Compare this key to another. Returns 0 if this key is equivalent to - the given key, or non-0 if they are different. Only the public parts - of the key are compared, so a public key will compare equal to its - corresponding private key. - - @param other: key to compare to. - @type other: L{PKey} - @return: 0 if the two keys are equivalent, non-0 otherwise. - @rtype: int - """ - hs = hash(self) - ho = hash(other) - if hs != ho: - return cmp(hs, ho) - return cmp(str(self), str(other)) - - def get_name(self): - """ - Return the name of this private key implementation. - - @return: name of this private key type, in SSH terminology (for - example, C{"ssh-rsa"}). - @rtype: str - """ - return '' - - def get_bits(self): - """ - Return the number of significant bits in this key. This is useful - for judging the relative security of a key. - - @return: bits in the key. - @rtype: int - """ - return 0 - - def can_sign(self): - """ - Return C{True} if this key has the private part necessary for signing - data. - - @return: C{True} if this is a private key. - @rtype: bool - """ - return False - - def get_fingerprint(self): - """ - Return an MD5 fingerprint of the public part of this key. Nothing - secret is revealed. - - @return: a 16-byte string (binary) of the MD5 fingerprint, in SSH - format. - @rtype: str - """ - return MD5.new(str(self)).digest() - - def get_base64(self): - """ - Return a base64 string containing the public part of this key. Nothing - secret is revealed. This format is compatible with that used to store - public key files or recognized host keys. - - @return: a base64 string containing the public part of the key. - @rtype: str - """ - return base64.encodestring(str(self)).replace('\n', '') - - def sign_ssh_data(self, rng, data): - """ - Sign a blob of data with this private key, and return a L{Message} - representing an SSH signature message. - - @param rng: a secure random number generator. - @type rng: L{Crypto.Util.rng.RandomPool} - @param data: the data to sign. - @type data: str - @return: an SSH signature message. - @rtype: L{Message} - """ - return '' - - def verify_ssh_sig(self, data, msg): - """ - Given a blob of data, and an SSH message representing a signature of - that data, verify that it was signed with this key. - - @param data: the data that was signed. - @type data: str - @param msg: an SSH signature message - @type msg: L{Message} - @return: C{True} if the signature verifies correctly; C{False} - otherwise. - @rtype: boolean - """ - return False - - def from_private_key_file(cls, filename, password=None): - """ - Create a key object by reading a private key file. If the private - key is encrypted and C{password} is not C{None}, the given password - will be used to decrypt the key (otherwise L{PasswordRequiredException} - is thrown). Through the magic of python, this factory method will - exist in all subclasses of PKey (such as L{RSAKey} or L{DSSKey}), but - is useless on the abstract PKey class. - - @param filename: name of the file to read - @type filename: str - @param password: an optional password to use to decrypt the key file, - if it's encrypted - @type password: str - @return: a new key object based on the given private key - @rtype: L{PKey} - - @raise IOError: if there was an error reading the file - @raise PasswordRequiredException: if the private key file is - encrypted, and C{password} is C{None} - @raise SSHException: if the key file is invalid - """ - key = cls(filename=filename, password=password) - return key - from_private_key_file = classmethod(from_private_key_file) - - def from_private_key(cls, file_obj, password=None): - """ - Create a key object by reading a private key from a file (or file-like) - object. If the private key is encrypted and C{password} is not C{None}, - the given password will be used to decrypt the key (otherwise - L{PasswordRequiredException} is thrown). - - @param file_obj: the file to read from - @type file_obj: file - @param password: an optional password to use to decrypt the key, if it's - encrypted - @type password: str - @return: a new key object based on the given private key - @rtype: L{PKey} - - @raise IOError: if there was an error reading the key - @raise PasswordRequiredException: if the private key file is encrypted, - and C{password} is C{None} - @raise SSHException: if the key file is invalid - """ - key = cls(file_obj=file_obj, password=password) - return key - from_private_key = classmethod(from_private_key) - - def write_private_key_file(self, filename, password=None): - """ - Write private key contents into a file. If the password is not - C{None}, the key is encrypted before writing. - - @param filename: name of the file to write - @type filename: str - @param password: an optional password to use to encrypt the key file - @type password: str - - @raise IOError: if there was an error writing the file - @raise SSHException: if the key is invalid - """ - raise Exception('Not implemented in PKey') - - def write_private_key(self, file_obj, password=None): - """ - Write private key contents into a file (or file-like) object. If the - password is not C{None}, the key is encrypted before writing. - - @param file_obj: the file object to write into - @type file_obj: file - @param password: an optional password to use to encrypt the key - @type password: str - - @raise IOError: if there was an error writing to the file - @raise SSHException: if the key is invalid - """ - raise Exception('Not implemented in PKey') - - def _read_private_key_file(self, tag, filename, password=None): - """ - Read an SSH2-format private key file, looking for a string of the type - C{"BEGIN xxx PRIVATE KEY"} for some C{xxx}, base64-decode the text we - find, and return it as a string. If the private key is encrypted and - C{password} is not C{None}, the given password will be used to decrypt - the key (otherwise L{PasswordRequiredException} is thrown). - - @param tag: C{"RSA"} or C{"DSA"}, the tag used to mark the data block. - @type tag: str - @param filename: name of the file to read. - @type filename: str - @param password: an optional password to use to decrypt the key file, - if it's encrypted. - @type password: str - @return: data blob that makes up the private key. - @rtype: str - - @raise IOError: if there was an error reading the file. - @raise PasswordRequiredException: if the private key file is - encrypted, and C{password} is C{None}. - @raise SSHException: if the key file is invalid. - """ - f = open(filename, 'r') - data = self._read_private_key(tag, f, password) - f.close() - return data - - def _read_private_key(self, tag, f, password=None): - lines = f.readlines() - start = 0 - while (start < len(lines)) and (lines[start].strip() != '-----BEGIN ' + tag + ' PRIVATE KEY-----'): - start += 1 - if start >= len(lines): - raise SSHException('not a valid ' + tag + ' private key file') - # parse any headers first - headers = {} - start += 1 - while start < len(lines): - l = lines[start].split(': ') - if len(l) == 1: - break - headers[l[0].lower()] = l[1].strip() - start += 1 - # find end - end = start - while (lines[end].strip() != '-----END ' + tag + ' PRIVATE KEY-----') and (end < len(lines)): - end += 1 - # if we trudged to the end of the file, just try to cope. - try: - data = base64.decodestring(''.join(lines[start:end])) - except base64.binascii.Error, e: - raise SSHException('base64 decoding error: ' + str(e)) - if 'proc-type' not in headers: - # unencryped: done - return data - # encrypted keyfile: will need a password - if headers['proc-type'] != '4,ENCRYPTED': - raise SSHException('Unknown private key structure "%s"' % headers['proc-type']) - try: - encryption_type, saltstr = headers['dek-info'].split(',') - except: - raise SSHException('Can\'t parse DEK-info in private key file') - if encryption_type not in self._CIPHER_TABLE: - raise SSHException('Unknown private key cipher "%s"' % encryption_type) - # if no password was passed in, raise an exception pointing out that we need one - if password is None: - raise PasswordRequiredException('Private key file is encrypted') - cipher = self._CIPHER_TABLE[encryption_type]['cipher'] - keysize = self._CIPHER_TABLE[encryption_type]['keysize'] - mode = self._CIPHER_TABLE[encryption_type]['mode'] - salt = unhexlify(saltstr) - key = util.generate_key_bytes(MD5, salt, password, keysize) - return cipher.new(key, mode, salt).decrypt(data) - - def _write_private_key_file(self, tag, filename, data, password=None): - """ - Write an SSH2-format private key file in a form that can be read by - paramiko or openssh. If no password is given, the key is written in - a trivially-encoded format (base64) which is completely insecure. If - a password is given, DES-EDE3-CBC is used. - - @param tag: C{"RSA"} or C{"DSA"}, the tag used to mark the data block. - @type tag: str - @param filename: name of the file to write. - @type filename: str - @param data: data blob that makes up the private key. - @type data: str - @param password: an optional password to use to encrypt the file. - @type password: str - - @raise IOError: if there was an error writing the file. - """ - f = open(filename, 'w', 0600) - # grrr... the mode doesn't always take hold - os.chmod(filename, 0600) - self._write_private_key(tag, f, data, password) - f.close() - - def _write_private_key(self, tag, f, data, password=None): - f.write('-----BEGIN %s PRIVATE KEY-----\n' % tag) - if password is not None: - # since we only support one cipher here, use it - cipher_name = self._CIPHER_TABLE.keys()[0] - cipher = self._CIPHER_TABLE[cipher_name]['cipher'] - keysize = self._CIPHER_TABLE[cipher_name]['keysize'] - blocksize = self._CIPHER_TABLE[cipher_name]['blocksize'] - mode = self._CIPHER_TABLE[cipher_name]['mode'] - salt = rng.read(8) - key = util.generate_key_bytes(MD5, salt, password, keysize) - if len(data) % blocksize != 0: - n = blocksize - len(data) % blocksize - #data += rng.read(n) - # that would make more sense ^, but it confuses openssh. - data += '\0' * n - data = cipher.new(key, mode, salt).encrypt(data) - f.write('Proc-Type: 4,ENCRYPTED\n') - f.write('DEK-Info: %s,%s\n' % (cipher_name, hexlify(salt).upper())) - f.write('\n') - s = base64.encodestring(data) - # re-wrap to 64-char lines - s = ''.join(s.split('\n')) - s = '\n'.join([s[i : i+64] for i in range(0, len(s), 64)]) - f.write(s) - f.write('\n') - f.write('-----END %s PRIVATE KEY-----\n' % tag) diff --git a/contrib/site-packages/paramiko/primes.py b/contrib/site-packages/paramiko/primes.py deleted file mode 100644 index 9419cd6b..00000000 --- a/contrib/site-packages/paramiko/primes.py +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -Utility functions for dealing with primes. -""" - -from Crypto.Util import number - -from paramiko import util -from paramiko.ssh_exception import SSHException - - -def _generate_prime(bits, rng): - "primtive attempt at prime generation" - hbyte_mask = pow(2, bits % 8) - 1 - while True: - # loop catches the case where we increment n into a higher bit-range - x = rng.read((bits+7) // 8) - if hbyte_mask > 0: - x = chr(ord(x[0]) & hbyte_mask) + x[1:] - n = util.inflate_long(x, 1) - n |= 1 - n |= (1 << (bits - 1)) - while not number.isPrime(n): - n += 2 - if util.bit_length(n) == bits: - break - return n - -def _roll_random(rng, n): - "returns a random # from 0 to N-1" - bits = util.bit_length(n-1) - bytes = (bits + 7) // 8 - hbyte_mask = pow(2, bits % 8) - 1 - - # so here's the plan: - # we fetch as many random bits as we'd need to fit N-1, and if the - # generated number is >= N, we try again. in the worst case (N-1 is a - # power of 2), we have slightly better than 50% odds of getting one that - # fits, so i can't guarantee that this loop will ever finish, but the odds - # of it looping forever should be infinitesimal. - while True: - x = rng.read(bytes) - if hbyte_mask > 0: - x = chr(ord(x[0]) & hbyte_mask) + x[1:] - num = util.inflate_long(x, 1) - if num < n: - break - return num - - -class ModulusPack (object): - """ - convenience object for holding the contents of the /etc/ssh/moduli file, - on systems that have such a file. - """ - - def __init__(self, rpool): - # pack is a hash of: bits -> [ (generator, modulus) ... ] - self.pack = {} - self.discarded = [] - self.rng = rpool - - def _parse_modulus(self, line): - timestamp, mod_type, tests, tries, size, generator, modulus = line.split() - mod_type = int(mod_type) - tests = int(tests) - tries = int(tries) - size = int(size) - generator = int(generator) - modulus = long(modulus, 16) - - # weed out primes that aren't at least: - # type 2 (meets basic structural requirements) - # test 4 (more than just a small-prime sieve) - # tries < 100 if test & 4 (at least 100 tries of miller-rabin) - if (mod_type < 2) or (tests < 4) or ((tests & 4) and (tests < 8) and (tries < 100)): - self.discarded.append((modulus, 'does not meet basic requirements')) - return - if generator == 0: - generator = 2 - - # there's a bug in the ssh "moduli" file (yeah, i know: shock! dismay! - # call cnn!) where it understates the bit lengths of these primes by 1. - # this is okay. - bl = util.bit_length(modulus) - if (bl != size) and (bl != size + 1): - self.discarded.append((modulus, 'incorrectly reported bit length %d' % size)) - return - if bl not in self.pack: - self.pack[bl] = [] - self.pack[bl].append((generator, modulus)) - - def read_file(self, filename): - """ - @raise IOError: passed from any file operations that fail. - """ - self.pack = {} - f = open(filename, 'r') - for line in f: - line = line.strip() - if (len(line) == 0) or (line[0] == '#'): - continue - try: - self._parse_modulus(line) - except: - continue - f.close() - - def get_modulus(self, min, prefer, max): - bitsizes = self.pack.keys() - bitsizes.sort() - if len(bitsizes) == 0: - raise SSHException('no moduli available') - good = -1 - # find nearest bitsize >= preferred - for b in bitsizes: - if (b >= prefer) and (b < max) and ((b < good) or (good == -1)): - good = b - # if that failed, find greatest bitsize >= min - if good == -1: - for b in bitsizes: - if (b >= min) and (b < max) and (b > good): - good = b - if good == -1: - # their entire (min, max) range has no intersection with our range. - # if their range is below ours, pick the smallest. otherwise pick - # the largest. it'll be out of their range requirement either way, - # but we'll be sending them the closest one we have. - good = bitsizes[0] - if min > good: - good = bitsizes[-1] - # now pick a random modulus of this bitsize - n = _roll_random(self.rng, len(self.pack[good])) - return self.pack[good][n] diff --git a/contrib/site-packages/paramiko/proxy.py b/contrib/site-packages/paramiko/proxy.py deleted file mode 100644 index 218b76e2..00000000 --- a/contrib/site-packages/paramiko/proxy.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (C) 2012 Yipit, Inc -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distrubuted in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -L{ProxyCommand}. -""" - -import os -from shlex import split as shlsplit -import signal -from subprocess import Popen, PIPE - -from paramiko.ssh_exception import ProxyCommandFailure - - -class ProxyCommand(object): - """ - Wraps a subprocess running ProxyCommand-driven programs. - - This class implements a the socket-like interface needed by the - L{Transport} and L{Packetizer} classes. Using this class instead of a - regular socket makes it possible to talk with a Popen'd command that will - proxy traffic between the client and a server hosted in another machine. - """ - def __init__(self, command_line): - """ - Create a new CommandProxy instance. The instance created by this - class can be passed as an argument to the L{Transport} class. - - @param command_line: the command that should be executed and - used as the proxy. - @type command_line: str - """ - self.cmd = shlsplit(command_line) - self.process = Popen(self.cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE) - - def send(self, content): - """ - Write the content received from the SSH client to the standard - input of the forked command. - - @param content: string to be sent to the forked command - @type content: str - """ - try: - self.process.stdin.write(content) - except IOError, e: - # There was a problem with the child process. It probably - # died and we can't proceed. The best option here is to - # raise an exception informing the user that the informed - # ProxyCommand is not working. - raise BadProxyCommand(' '.join(self.cmd), e.strerror) - return len(content) - - def recv(self, size): - """ - Read from the standard output of the forked program. - - @param size: how many chars should be read - @type size: int - - @return: the length of the read content - @rtype: int - """ - try: - return os.read(self.process.stdout.fileno(), size) - except IOError, e: - raise BadProxyCommand(' '.join(self.cmd), e.strerror) - - def close(self): - os.kill(self.process.pid, signal.SIGTERM) - - def settimeout(self, timeout): - # Timeouts are meaningless for this implementation, but are part of the - # spec, so must be present. - pass diff --git a/contrib/site-packages/paramiko/resource.py b/contrib/site-packages/paramiko/resource.py deleted file mode 100644 index 6ef86d8c..00000000 --- a/contrib/site-packages/paramiko/resource.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -Resource manager. -""" - -import weakref - - -class ResourceManager (object): - """ - A registry of objects and resources that should be closed when those - objects are deleted. - - This is meant to be a safer alternative to python's C{__del__} method, - which can cause reference cycles to never be collected. Objects registered - with the ResourceManager can be collected but still free resources when - they die. - - Resources are registered using L{register}, and when an object is garbage - collected, each registered resource is closed by having its C{close()} - method called. Multiple resources may be registered per object, but a - resource will only be closed once, even if multiple objects register it. - (The last object to register it wins.) - """ - - def __init__(self): - self._table = {} - - def register(self, obj, resource): - """ - Register a resource to be closed with an object is collected. - - When the given C{obj} is garbage-collected by the python interpreter, - the C{resource} will be closed by having its C{close()} method called. - Any exceptions are ignored. - - @param obj: the object to track - @type obj: object - @param resource: the resource to close when the object is collected - @type resource: object - """ - def callback(ref): - try: - resource.close() - except: - pass - del self._table[id(resource)] - - # keep the weakref in a table so it sticks around long enough to get - # its callback called. :) - self._table[id(resource)] = weakref.ref(obj, callback) - - -# singleton -ResourceManager = ResourceManager() diff --git a/contrib/site-packages/paramiko/rsakey.py b/contrib/site-packages/paramiko/rsakey.py deleted file mode 100644 index c7500f85..00000000 --- a/contrib/site-packages/paramiko/rsakey.py +++ /dev/null @@ -1,185 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -L{RSAKey} -""" - -from Crypto.PublicKey import RSA -from Crypto.Hash import SHA, MD5 -from Crypto.Cipher import DES3 - -from paramiko.common import * -from paramiko import util -from paramiko.message import Message -from paramiko.ber import BER, BERException -from paramiko.pkey import PKey -from paramiko.ssh_exception import SSHException - - -class RSAKey (PKey): - """ - Representation of an RSA key which can be used to sign and verify SSH2 - data. - """ - - def __init__(self, msg=None, data=None, filename=None, password=None, vals=None, file_obj=None): - self.n = None - self.e = None - self.d = None - self.p = None - self.q = None - if file_obj is not None: - self._from_private_key(file_obj, password) - return - if filename is not None: - self._from_private_key_file(filename, password) - return - if (msg is None) and (data is not None): - msg = Message(data) - if vals is not None: - self.e, self.n = vals - else: - if msg is None: - raise SSHException('Key object may not be empty') - if msg.get_string() != 'ssh-rsa': - raise SSHException('Invalid key') - self.e = msg.get_mpint() - self.n = msg.get_mpint() - self.size = util.bit_length(self.n) - - def __str__(self): - m = Message() - m.add_string('ssh-rsa') - m.add_mpint(self.e) - m.add_mpint(self.n) - return str(m) - - def __hash__(self): - h = hash(self.get_name()) - h = h * 37 + hash(self.e) - h = h * 37 + hash(self.n) - return hash(h) - - def get_name(self): - return 'ssh-rsa' - - def get_bits(self): - return self.size - - def can_sign(self): - return self.d is not None - - def sign_ssh_data(self, rpool, data): - digest = SHA.new(data).digest() - rsa = RSA.construct((long(self.n), long(self.e), long(self.d))) - sig = util.deflate_long(rsa.sign(self._pkcs1imify(digest), '')[0], 0) - m = Message() - m.add_string('ssh-rsa') - m.add_string(sig) - return m - - def verify_ssh_sig(self, data, msg): - if msg.get_string() != 'ssh-rsa': - return False - sig = util.inflate_long(msg.get_string(), True) - # verify the signature by SHA'ing the data and encrypting it using the - # public key. some wackiness ensues where we "pkcs1imify" the 20-byte - # hash into a string as long as the RSA key. - hash_obj = util.inflate_long(self._pkcs1imify(SHA.new(data).digest()), True) - rsa = RSA.construct((long(self.n), long(self.e))) - return rsa.verify(hash_obj, (sig,)) - - def _encode_key(self): - if (self.p is None) or (self.q is None): - raise SSHException('Not enough key info to write private key file') - keylist = [ 0, self.n, self.e, self.d, self.p, self.q, - self.d % (self.p - 1), self.d % (self.q - 1), - util.mod_inverse(self.q, self.p) ] - try: - b = BER() - b.encode(keylist) - except BERException: - raise SSHException('Unable to create ber encoding of key') - return str(b) - - def write_private_key_file(self, filename, password=None): - self._write_private_key_file('RSA', filename, self._encode_key(), password) - - def write_private_key(self, file_obj, password=None): - self._write_private_key('RSA', file_obj, self._encode_key(), password) - - def generate(bits, progress_func=None): - """ - Generate a new private RSA key. This factory function can be used to - generate a new host key or authentication key. - - @param bits: number of bits the generated key should be. - @type bits: int - @param progress_func: an optional function to call at key points in - key generation (used by C{pyCrypto.PublicKey}). - @type progress_func: function - @return: new private key - @rtype: L{RSAKey} - """ - rsa = RSA.generate(bits, rng.read, progress_func) - key = RSAKey(vals=(rsa.e, rsa.n)) - key.d = rsa.d - key.p = rsa.p - key.q = rsa.q - return key - generate = staticmethod(generate) - - - ### internals... - - - def _pkcs1imify(self, data): - """ - turn a 20-byte SHA1 hash into a blob of data as large as the key's N, - using PKCS1's \"emsa-pkcs1-v1_5\" encoding. totally bizarre. - """ - SHA1_DIGESTINFO = '\x30\x21\x30\x09\x06\x05\x2b\x0e\x03\x02\x1a\x05\x00\x04\x14' - size = len(util.deflate_long(self.n, 0)) - filler = '\xff' * (size - len(SHA1_DIGESTINFO) - len(data) - 3) - return '\x00\x01' + filler + '\x00' + SHA1_DIGESTINFO + data - - def _from_private_key_file(self, filename, password): - data = self._read_private_key_file('RSA', filename, password) - self._decode_key(data) - - def _from_private_key(self, file_obj, password): - data = self._read_private_key('RSA', file_obj, password) - self._decode_key(data) - - def _decode_key(self, data): - # private key file contains: - # RSAPrivateKey = { version = 0, n, e, d, p, q, d mod p-1, d mod q-1, q**-1 mod p } - try: - keylist = BER(data).decode() - except BERException: - raise SSHException('Unable to parse key file') - if (type(keylist) is not list) or (len(keylist) < 4) or (keylist[0] != 0): - raise SSHException('Not a valid RSA private key file (bad ber encoding)') - self.n = keylist[1] - self.e = keylist[2] - self.d = keylist[3] - # not really needed - self.p = keylist[4] - self.q = keylist[5] - self.size = util.bit_length(self.n) diff --git a/contrib/site-packages/paramiko/server.py b/contrib/site-packages/paramiko/server.py deleted file mode 100644 index fdb40942..00000000 --- a/contrib/site-packages/paramiko/server.py +++ /dev/null @@ -1,669 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -L{ServerInterface} is an interface to override for server support. -""" - -import threading -from paramiko.common import * -from paramiko import util - - -class InteractiveQuery (object): - """ - A query (set of prompts) for a user during interactive authentication. - """ - - def __init__(self, name='', instructions='', *prompts): - """ - Create a new interactive query to send to the client. The name and - instructions are optional, but are generally displayed to the end - user. A list of prompts may be included, or they may be added via - the L{add_prompt} method. - - @param name: name of this query - @type name: str - @param instructions: user instructions (usually short) about this query - @type instructions: str - @param prompts: one or more authentication prompts - @type prompts: str - """ - self.name = name - self.instructions = instructions - self.prompts = [] - for x in prompts: - if (type(x) is str) or (type(x) is unicode): - self.add_prompt(x) - else: - self.add_prompt(x[0], x[1]) - - def add_prompt(self, prompt, echo=True): - """ - Add a prompt to this query. The prompt should be a (reasonably short) - string. Multiple prompts can be added to the same query. - - @param prompt: the user prompt - @type prompt: str - @param echo: C{True} (default) if the user's response should be echoed; - C{False} if not (for a password or similar) - @type echo: bool - """ - self.prompts.append((prompt, echo)) - - -class ServerInterface (object): - """ - This class defines an interface for controlling the behavior of paramiko - in server mode. - - Methods on this class are called from paramiko's primary thread, so you - shouldn't do too much work in them. (Certainly nothing that blocks or - sleeps.) - """ - - def check_channel_request(self, kind, chanid): - """ - Determine if a channel request of a given type will be granted, and - return C{OPEN_SUCCEEDED} or an error code. This method is - called in server mode when the client requests a channel, after - authentication is complete. - - If you allow channel requests (and an ssh server that didn't would be - useless), you should also override some of the channel request methods - below, which are used to determine which services will be allowed on - a given channel: - - L{check_channel_pty_request} - - L{check_channel_shell_request} - - L{check_channel_subsystem_request} - - L{check_channel_window_change_request} - - L{check_channel_x11_request} - - L{check_channel_forward_agent_request} - - The C{chanid} parameter is a small number that uniquely identifies the - channel within a L{Transport}. A L{Channel} object is not created - unless this method returns C{OPEN_SUCCEEDED} -- once a - L{Channel} object is created, you can call L{Channel.get_id} to - retrieve the channel ID. - - The return value should either be C{OPEN_SUCCEEDED} (or - C{0}) to allow the channel request, or one of the following error - codes to reject it: - - C{OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED} - - C{OPEN_FAILED_CONNECT_FAILED} - - C{OPEN_FAILED_UNKNOWN_CHANNEL_TYPE} - - C{OPEN_FAILED_RESOURCE_SHORTAGE} - - The default implementation always returns - C{OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED}. - - @param kind: the kind of channel the client would like to open - (usually C{"session"}). - @type kind: str - @param chanid: ID of the channel - @type chanid: int - @return: a success or failure code (listed above) - @rtype: int - """ - return OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED - - def get_allowed_auths(self, username): - """ - Return a list of authentication methods supported by the server. - This list is sent to clients attempting to authenticate, to inform them - of authentication methods that might be successful. - - The "list" is actually a string of comma-separated names of types of - authentication. Possible values are C{"password"}, C{"publickey"}, - and C{"none"}. - - The default implementation always returns C{"password"}. - - @param username: the username requesting authentication. - @type username: str - @return: a comma-separated list of authentication types - @rtype: str - """ - return 'password' - - def check_auth_none(self, username): - """ - Determine if a client may open channels with no (further) - authentication. - - Return L{AUTH_FAILED} if the client must authenticate, or - L{AUTH_SUCCESSFUL} if it's okay for the client to not - authenticate. - - The default implementation always returns L{AUTH_FAILED}. - - @param username: the username of the client. - @type username: str - @return: L{AUTH_FAILED} if the authentication fails; - L{AUTH_SUCCESSFUL} if it succeeds. - @rtype: int - """ - return AUTH_FAILED - - def check_auth_password(self, username, password): - """ - Determine if a given username and password supplied by the client is - acceptable for use in authentication. - - Return L{AUTH_FAILED} if the password is not accepted, - L{AUTH_SUCCESSFUL} if the password is accepted and completes - the authentication, or L{AUTH_PARTIALLY_SUCCESSFUL} if your - authentication is stateful, and this key is accepted for - authentication, but more authentication is required. (In this latter - case, L{get_allowed_auths} will be called to report to the client what - options it has for continuing the authentication.) - - The default implementation always returns L{AUTH_FAILED}. - - @param username: the username of the authenticating client. - @type username: str - @param password: the password given by the client. - @type password: str - @return: L{AUTH_FAILED} if the authentication fails; - L{AUTH_SUCCESSFUL} if it succeeds; - L{AUTH_PARTIALLY_SUCCESSFUL} if the password auth is - successful, but authentication must continue. - @rtype: int - """ - return AUTH_FAILED - - def check_auth_publickey(self, username, key): - """ - Determine if a given key supplied by the client is acceptable for use - in authentication. You should override this method in server mode to - check the username and key and decide if you would accept a signature - made using this key. - - Return L{AUTH_FAILED} if the key is not accepted, - L{AUTH_SUCCESSFUL} if the key is accepted and completes the - authentication, or L{AUTH_PARTIALLY_SUCCESSFUL} if your - authentication is stateful, and this password is accepted for - authentication, but more authentication is required. (In this latter - case, L{get_allowed_auths} will be called to report to the client what - options it has for continuing the authentication.) - - Note that you don't have to actually verify any key signtature here. - If you're willing to accept the key, paramiko will do the work of - verifying the client's signature. - - The default implementation always returns L{AUTH_FAILED}. - - @param username: the username of the authenticating client - @type username: str - @param key: the key object provided by the client - @type key: L{PKey } - @return: L{AUTH_FAILED} if the client can't authenticate - with this key; L{AUTH_SUCCESSFUL} if it can; - L{AUTH_PARTIALLY_SUCCESSFUL} if it can authenticate with - this key but must continue with authentication - @rtype: int - """ - return AUTH_FAILED - - def check_auth_interactive(self, username, submethods): - """ - Begin an interactive authentication challenge, if supported. You - should override this method in server mode if you want to support the - C{"keyboard-interactive"} auth type, which requires you to send a - series of questions for the client to answer. - - Return L{AUTH_FAILED} if this auth method isn't supported. Otherwise, - you should return an L{InteractiveQuery} object containing the prompts - and instructions for the user. The response will be sent via a call - to L{check_auth_interactive_response}. - - The default implementation always returns L{AUTH_FAILED}. - - @param username: the username of the authenticating client - @type username: str - @param submethods: a comma-separated list of methods preferred by the - client (usually empty) - @type submethods: str - @return: L{AUTH_FAILED} if this auth method isn't supported; otherwise - an object containing queries for the user - @rtype: int or L{InteractiveQuery} - """ - return AUTH_FAILED - - def check_auth_interactive_response(self, responses): - """ - Continue or finish an interactive authentication challenge, if - supported. You should override this method in server mode if you want - to support the C{"keyboard-interactive"} auth type. - - Return L{AUTH_FAILED} if the responses are not accepted, - L{AUTH_SUCCESSFUL} if the responses are accepted and complete - the authentication, or L{AUTH_PARTIALLY_SUCCESSFUL} if your - authentication is stateful, and this set of responses is accepted for - authentication, but more authentication is required. (In this latter - case, L{get_allowed_auths} will be called to report to the client what - options it has for continuing the authentication.) - - If you wish to continue interactive authentication with more questions, - you may return an L{InteractiveQuery} object, which should cause the - client to respond with more answers, calling this method again. This - cycle can continue indefinitely. - - The default implementation always returns L{AUTH_FAILED}. - - @param responses: list of responses from the client - @type responses: list(str) - @return: L{AUTH_FAILED} if the authentication fails; - L{AUTH_SUCCESSFUL} if it succeeds; - L{AUTH_PARTIALLY_SUCCESSFUL} if the interactive auth is - successful, but authentication must continue; otherwise an object - containing queries for the user - @rtype: int or L{InteractiveQuery} - """ - return AUTH_FAILED - - def check_port_forward_request(self, address, port): - """ - Handle a request for port forwarding. The client is asking that - connections to the given address and port be forwarded back across - this ssh connection. An address of C{"0.0.0.0"} indicates a global - address (any address associated with this server) and a port of C{0} - indicates that no specific port is requested (usually the OS will pick - a port). - - The default implementation always returns C{False}, rejecting the - port forwarding request. If the request is accepted, you should return - the port opened for listening. - - @param address: the requested address - @type address: str - @param port: the requested port - @type port: int - @return: the port number that was opened for listening, or C{False} to - reject - @rtype: int - """ - return False - - def cancel_port_forward_request(self, address, port): - """ - The client would like to cancel a previous port-forwarding request. - If the given address and port is being forwarded across this ssh - connection, the port should be closed. - - @param address: the forwarded address - @type address: str - @param port: the forwarded port - @type port: int - """ - pass - - def check_global_request(self, kind, msg): - """ - Handle a global request of the given C{kind}. This method is called - in server mode and client mode, whenever the remote host makes a global - request. If there are any arguments to the request, they will be in - C{msg}. - - There aren't any useful global requests defined, aside from port - forwarding, so usually this type of request is an extension to the - protocol. - - If the request was successful and you would like to return contextual - data to the remote host, return a tuple. Items in the tuple will be - sent back with the successful result. (Note that the items in the - tuple can only be strings, ints, longs, or bools.) - - The default implementation always returns C{False}, indicating that it - does not support any global requests. - - @note: Port forwarding requests are handled separately, in - L{check_port_forward_request}. - - @param kind: the kind of global request being made. - @type kind: str - @param msg: any extra arguments to the request. - @type msg: L{Message} - @return: C{True} or a tuple of data if the request was granted; - C{False} otherwise. - @rtype: bool - """ - return False - - - ### Channel requests - - - def check_channel_pty_request(self, channel, term, width, height, pixelwidth, pixelheight, - modes): - """ - Determine if a pseudo-terminal of the given dimensions (usually - requested for shell access) can be provided on the given channel. - - The default implementation always returns C{False}. - - @param channel: the L{Channel} the pty request arrived on. - @type channel: L{Channel} - @param term: type of terminal requested (for example, C{"vt100"}). - @type term: str - @param width: width of screen in characters. - @type width: int - @param height: height of screen in characters. - @type height: int - @param pixelwidth: width of screen in pixels, if known (may be C{0} if - unknown). - @type pixelwidth: int - @param pixelheight: height of screen in pixels, if known (may be C{0} - if unknown). - @type pixelheight: int - @return: C{True} if the psuedo-terminal has been allocated; C{False} - otherwise. - @rtype: bool - """ - return False - - def check_channel_shell_request(self, channel): - """ - Determine if a shell will be provided to the client on the given - channel. If this method returns C{True}, the channel should be - connected to the stdin/stdout of a shell (or something that acts like - a shell). - - The default implementation always returns C{False}. - - @param channel: the L{Channel} the request arrived on. - @type channel: L{Channel} - @return: C{True} if this channel is now hooked up to a shell; C{False} - if a shell can't or won't be provided. - @rtype: bool - """ - return False - - def check_channel_exec_request(self, channel, command): - """ - Determine if a shell command will be executed for the client. If this - method returns C{True}, the channel should be connected to the stdin, - stdout, and stderr of the shell command. - - The default implementation always returns C{False}. - - @param channel: the L{Channel} the request arrived on. - @type channel: L{Channel} - @param command: the command to execute. - @type command: str - @return: C{True} if this channel is now hooked up to the stdin, - stdout, and stderr of the executing command; C{False} if the - command will not be executed. - @rtype: bool - - @since: 1.1 - """ - return False - - def check_channel_subsystem_request(self, channel, name): - """ - Determine if a requested subsystem will be provided to the client on - the given channel. If this method returns C{True}, all future I/O - through this channel will be assumed to be connected to the requested - subsystem. An example of a subsystem is C{sftp}. - - The default implementation checks for a subsystem handler assigned via - L{Transport.set_subsystem_handler}. - If one has been set, the handler is invoked and this method returns - C{True}. Otherwise it returns C{False}. - - @note: Because the default implementation uses the L{Transport} to - identify valid subsystems, you probably won't need to override this - method. - - @param channel: the L{Channel} the pty request arrived on. - @type channel: L{Channel} - @param name: name of the requested subsystem. - @type name: str - @return: C{True} if this channel is now hooked up to the requested - subsystem; C{False} if that subsystem can't or won't be provided. - @rtype: bool - """ - handler_class, larg, kwarg = channel.get_transport()._get_subsystem_handler(name) - if handler_class is None: - return False - handler = handler_class(channel, name, self, *larg, **kwarg) - handler.start() - return True - - def check_channel_window_change_request(self, channel, width, height, pixelwidth, pixelheight): - """ - Determine if the pseudo-terminal on the given channel can be resized. - This only makes sense if a pty was previously allocated on it. - - The default implementation always returns C{False}. - - @param channel: the L{Channel} the pty request arrived on. - @type channel: L{Channel} - @param width: width of screen in characters. - @type width: int - @param height: height of screen in characters. - @type height: int - @param pixelwidth: width of screen in pixels, if known (may be C{0} if - unknown). - @type pixelwidth: int - @param pixelheight: height of screen in pixels, if known (may be C{0} - if unknown). - @type pixelheight: int - @return: C{True} if the terminal was resized; C{False} if not. - @rtype: bool - """ - return False - - def check_channel_x11_request(self, channel, single_connection, auth_protocol, auth_cookie, screen_number): - """ - Determine if the client will be provided with an X11 session. If this - method returns C{True}, X11 applications should be routed through new - SSH channels, using L{Transport.open_x11_channel}. - - The default implementation always returns C{False}. - - @param channel: the L{Channel} the X11 request arrived on - @type channel: L{Channel} - @param single_connection: C{True} if only a single X11 channel should - be opened - @type single_connection: bool - @param auth_protocol: the protocol used for X11 authentication - @type auth_protocol: str - @param auth_cookie: the cookie used to authenticate to X11 - @type auth_cookie: str - @param screen_number: the number of the X11 screen to connect to - @type screen_number: int - @return: C{True} if the X11 session was opened; C{False} if not - @rtype: bool - """ - return False - - def check_channel_forward_agent_request(self, channel): - """ - Determine if the client will be provided with an forward agent session. - If this method returns C{True}, the server will allow SSH Agent - forwarding. - - The default implementation always returns C{False}. - - @param channel: the L{Channel} the request arrived on - @type channel: L{Channel} - @return: C{True} if the AgentForward was loaded; C{False} if not - @rtype: bool - """ - return False - - def check_channel_direct_tcpip_request(self, chanid, origin, destination): - """ - Determine if a local port forwarding channel will be granted, and - return C{OPEN_SUCCEEDED} or an error code. This method is - called in server mode when the client requests a channel, after - authentication is complete. - - The C{chanid} parameter is a small number that uniquely identifies the - channel within a L{Transport}. A L{Channel} object is not created - unless this method returns C{OPEN_SUCCEEDED} -- once a - L{Channel} object is created, you can call L{Channel.get_id} to - retrieve the channel ID. - - The origin and destination parameters are (ip_address, port) tuples - that correspond to both ends of the TCP connection in the forwarding - tunnel. - - The return value should either be C{OPEN_SUCCEEDED} (or - C{0}) to allow the channel request, or one of the following error - codes to reject it: - - C{OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED} - - C{OPEN_FAILED_CONNECT_FAILED} - - C{OPEN_FAILED_UNKNOWN_CHANNEL_TYPE} - - C{OPEN_FAILED_RESOURCE_SHORTAGE} - - The default implementation always returns - C{OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED}. - - @param chanid: ID of the channel - @type chanid: int - @param origin: 2-tuple containing the IP address and port of the - originator (client side) - @type origin: tuple - @param destination: 2-tuple containing the IP address and port of the - destination (server side) - @type destination: tuple - @return: a success or failure code (listed above) - @rtype: int - """ - return OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED - - def check_channel_env_request(self, channel, name, value): - """ - Check whether a given environment variable can be specified for the - given channel. This method should return C{True} if the server - is willing to set the specified environment variable. Note that - some environment variables (e.g., PATH) can be exceedingly - dangerous, so blindly allowing the client to set the environment - is almost certainly not a good idea. - - The default implementation always returns C{False}. - - @param channel: the L{Channel} the env request arrived on - @type channel: L{Channel} - @param name: foo bar baz - @type name: str - @param value: flklj - @type value: str - @rtype: bool - """ - return False - - -class SubsystemHandler (threading.Thread): - """ - Handler for a subsytem in server mode. If you create a subclass of this - class and pass it to - L{Transport.set_subsystem_handler}, - an object of this - class will be created for each request for this subsystem. Each new object - will be executed within its own new thread by calling L{start_subsystem}. - When that method completes, the channel is closed. - - For example, if you made a subclass C{MP3Handler} and registered it as the - handler for subsystem C{"mp3"}, then whenever a client has successfully - authenticated and requests subsytem C{"mp3"}, an object of class - C{MP3Handler} will be created, and L{start_subsystem} will be called on - it from a new thread. - """ - def __init__(self, channel, name, server): - """ - Create a new handler for a channel. This is used by L{ServerInterface} - to start up a new handler when a channel requests this subsystem. You - don't need to override this method, but if you do, be sure to pass the - C{channel} and C{name} parameters through to the original C{__init__} - method here. - - @param channel: the channel associated with this subsystem request. - @type channel: L{Channel} - @param name: name of the requested subsystem. - @type name: str - @param server: the server object for the session that started this - subsystem - @type server: L{ServerInterface} - """ - threading.Thread.__init__(self, target=self._run) - self.__channel = channel - self.__transport = channel.get_transport() - self.__name = name - self.__server = server - - def get_server(self): - """ - Return the L{ServerInterface} object associated with this channel and - subsystem. - - @rtype: L{ServerInterface} - """ - return self.__server - - def _run(self): - try: - self.__transport._log(DEBUG, 'Starting handler for subsystem %s' % self.__name) - self.start_subsystem(self.__name, self.__transport, self.__channel) - except Exception, e: - self.__transport._log(ERROR, 'Exception in subsystem handler for "%s": %s' % - (self.__name, str(e))) - self.__transport._log(ERROR, util.tb_strings()) - try: - self.finish_subsystem() - except: - pass - - def start_subsystem(self, name, transport, channel): - """ - Process an ssh subsystem in server mode. This method is called on a - new object (and in a new thread) for each subsystem request. It is - assumed that all subsystem logic will take place here, and when the - subsystem is finished, this method will return. After this method - returns, the channel is closed. - - The combination of C{transport} and C{channel} are unique; this handler - corresponds to exactly one L{Channel} on one L{Transport}. - - @note: It is the responsibility of this method to exit if the - underlying L{Transport} is closed. This can be done by checking - L{Transport.is_active} or noticing an EOF - on the L{Channel}. If this method loops forever without checking - for this case, your python interpreter may refuse to exit because - this thread will still be running. - - @param name: name of the requested subsystem. - @type name: str - @param transport: the server-mode L{Transport}. - @type transport: L{Transport} - @param channel: the channel associated with this subsystem request. - @type channel: L{Channel} - """ - pass - - def finish_subsystem(self): - """ - Perform any cleanup at the end of a subsystem. The default - implementation just closes the channel. - - @since: 1.1 - """ - self.__channel.close() diff --git a/contrib/site-packages/paramiko/sftp.py b/contrib/site-packages/paramiko/sftp.py deleted file mode 100644 index a97c300f..00000000 --- a/contrib/site-packages/paramiko/sftp.py +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -import select -import socket -import struct - -from paramiko.common import * -from paramiko import util -from paramiko.channel import Channel -from paramiko.message import Message - - -CMD_INIT, CMD_VERSION, CMD_OPEN, CMD_CLOSE, CMD_READ, CMD_WRITE, CMD_LSTAT, CMD_FSTAT, \ - CMD_SETSTAT, CMD_FSETSTAT, CMD_OPENDIR, CMD_READDIR, CMD_REMOVE, CMD_MKDIR, \ - CMD_RMDIR, CMD_REALPATH, CMD_STAT, CMD_RENAME, CMD_READLINK, CMD_SYMLINK \ - = range(1, 21) -CMD_STATUS, CMD_HANDLE, CMD_DATA, CMD_NAME, CMD_ATTRS = range(101, 106) -CMD_EXTENDED, CMD_EXTENDED_REPLY = range(200, 202) - -SFTP_OK = 0 -SFTP_EOF, SFTP_NO_SUCH_FILE, SFTP_PERMISSION_DENIED, SFTP_FAILURE, SFTP_BAD_MESSAGE, \ - SFTP_NO_CONNECTION, SFTP_CONNECTION_LOST, SFTP_OP_UNSUPPORTED = range(1, 9) - -SFTP_DESC = [ 'Success', - 'End of file', - 'No such file', - 'Permission denied', - 'Failure', - 'Bad message', - 'No connection', - 'Connection lost', - 'Operation unsupported' ] - -SFTP_FLAG_READ = 0x1 -SFTP_FLAG_WRITE = 0x2 -SFTP_FLAG_APPEND = 0x4 -SFTP_FLAG_CREATE = 0x8 -SFTP_FLAG_TRUNC = 0x10 -SFTP_FLAG_EXCL = 0x20 - -_VERSION = 3 - - -# for debugging -CMD_NAMES = { - CMD_INIT: 'init', - CMD_VERSION: 'version', - CMD_OPEN: 'open', - CMD_CLOSE: 'close', - CMD_READ: 'read', - CMD_WRITE: 'write', - CMD_LSTAT: 'lstat', - CMD_FSTAT: 'fstat', - CMD_SETSTAT: 'setstat', - CMD_FSETSTAT: 'fsetstat', - CMD_OPENDIR: 'opendir', - CMD_READDIR: 'readdir', - CMD_REMOVE: 'remove', - CMD_MKDIR: 'mkdir', - CMD_RMDIR: 'rmdir', - CMD_REALPATH: 'realpath', - CMD_STAT: 'stat', - CMD_RENAME: 'rename', - CMD_READLINK: 'readlink', - CMD_SYMLINK: 'symlink', - CMD_STATUS: 'status', - CMD_HANDLE: 'handle', - CMD_DATA: 'data', - CMD_NAME: 'name', - CMD_ATTRS: 'attrs', - CMD_EXTENDED: 'extended', - CMD_EXTENDED_REPLY: 'extended_reply' - } - - -class SFTPError (Exception): - pass - - -class BaseSFTP (object): - def __init__(self): - self.logger = util.get_logger('paramiko.sftp') - self.sock = None - self.ultra_debug = False - - - ### internals... - - - def _send_version(self): - self._send_packet(CMD_INIT, struct.pack('>I', _VERSION)) - t, data = self._read_packet() - if t != CMD_VERSION: - raise SFTPError('Incompatible sftp protocol') - version = struct.unpack('>I', data[:4])[0] - # if version != _VERSION: - # raise SFTPError('Incompatible sftp protocol') - return version - - def _send_server_version(self): - # winscp will freak out if the server sends version info before the - # client finishes sending INIT. - t, data = self._read_packet() - if t != CMD_INIT: - raise SFTPError('Incompatible sftp protocol') - version = struct.unpack('>I', data[:4])[0] - # advertise that we support "check-file" - extension_pairs = [ 'check-file', 'md5,sha1' ] - msg = Message() - msg.add_int(_VERSION) - msg.add(*extension_pairs) - self._send_packet(CMD_VERSION, str(msg)) - return version - - def _log(self, level, msg, *args): - self.logger.log(level, msg, *args) - - def _write_all(self, out): - while len(out) > 0: - n = self.sock.send(out) - if n <= 0: - raise EOFError() - if n == len(out): - return - out = out[n:] - return - - def _read_all(self, n): - out = '' - while n > 0: - if isinstance(self.sock, socket.socket): - # sometimes sftp is used directly over a socket instead of - # through a paramiko channel. in this case, check periodically - # if the socket is closed. (for some reason, recv() won't ever - # return or raise an exception, but calling select on a closed - # socket will.) - while True: - read, write, err = select.select([ self.sock ], [], [], 0.1) - if len(read) > 0: - x = self.sock.recv(n) - break - else: - x = self.sock.recv(n) - - if len(x) == 0: - raise EOFError() - out += x - n -= len(x) - return out - - def _send_packet(self, t, packet): - #self._log(DEBUG2, 'write: %s (len=%d)' % (CMD_NAMES.get(t, '0x%02x' % t), len(packet))) - out = struct.pack('>I', len(packet) + 1) + chr(t) + packet - if self.ultra_debug: - self._log(DEBUG, util.format_binary(out, 'OUT: ')) - self._write_all(out) - - def _read_packet(self): - x = self._read_all(4) - # most sftp servers won't accept packets larger than about 32k, so - # anything with the high byte set (> 16MB) is just garbage. - if x[0] != '\x00': - raise SFTPError('Garbage packet received') - size = struct.unpack('>I', x)[0] - data = self._read_all(size) - if self.ultra_debug: - self._log(DEBUG, util.format_binary(data, 'IN: ')); - if size > 0: - t = ord(data[0]) - #self._log(DEBUG2, 'read: %s (len=%d)' % (CMD_NAMES.get(t), '0x%02x' % t, len(data)-1)) - return t, data[1:] - return 0, '' diff --git a/contrib/site-packages/paramiko/sftp_attr.py b/contrib/site-packages/paramiko/sftp_attr.py deleted file mode 100644 index b459b04b..00000000 --- a/contrib/site-packages/paramiko/sftp_attr.py +++ /dev/null @@ -1,223 +0,0 @@ -# Copyright (C) 2003-2006 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -import stat -import time -from paramiko.common import * -from paramiko.sftp import * - - -class SFTPAttributes (object): - """ - Representation of the attributes of a file (or proxied file) for SFTP in - client or server mode. It attemps to mirror the object returned by - C{os.stat} as closely as possible, so it may have the following fields, - with the same meanings as those returned by an C{os.stat} object: - - st_size - - st_uid - - st_gid - - st_mode - - st_atime - - st_mtime - - Because SFTP allows flags to have other arbitrary named attributes, these - are stored in a dict named C{attr}. Occasionally, the filename is also - stored, in C{filename}. - """ - - FLAG_SIZE = 1 - FLAG_UIDGID = 2 - FLAG_PERMISSIONS = 4 - FLAG_AMTIME = 8 - FLAG_EXTENDED = 0x80000000L - - def __init__(self): - """ - Create a new (empty) SFTPAttributes object. All fields will be empty. - """ - self._flags = 0 - self.st_size = None - self.st_uid = None - self.st_gid = None - self.st_mode = None - self.st_atime = None - self.st_mtime = None - self.attr = {} - - def from_stat(cls, obj, filename=None): - """ - Create an SFTPAttributes object from an existing C{stat} object (an - object returned by C{os.stat}). - - @param obj: an object returned by C{os.stat} (or equivalent). - @type obj: object - @param filename: the filename associated with this file. - @type filename: str - @return: new L{SFTPAttributes} object with the same attribute fields. - @rtype: L{SFTPAttributes} - """ - attr = cls() - attr.st_size = obj.st_size - attr.st_uid = obj.st_uid - attr.st_gid = obj.st_gid - attr.st_mode = obj.st_mode - attr.st_atime = obj.st_atime - attr.st_mtime = obj.st_mtime - if filename is not None: - attr.filename = filename - return attr - from_stat = classmethod(from_stat) - - def __repr__(self): - return '' % self._debug_str() - - - ### internals... - - - def _from_msg(cls, msg, filename=None, longname=None): - attr = cls() - attr._unpack(msg) - if filename is not None: - attr.filename = filename - if longname is not None: - attr.longname = longname - return attr - _from_msg = classmethod(_from_msg) - - def _unpack(self, msg): - self._flags = msg.get_int() - if self._flags & self.FLAG_SIZE: - self.st_size = msg.get_int64() - if self._flags & self.FLAG_UIDGID: - self.st_uid = msg.get_int() - self.st_gid = msg.get_int() - if self._flags & self.FLAG_PERMISSIONS: - self.st_mode = msg.get_int() - if self._flags & self.FLAG_AMTIME: - self.st_atime = msg.get_int() - self.st_mtime = msg.get_int() - if self._flags & self.FLAG_EXTENDED: - count = msg.get_int() - for i in range(count): - self.attr[msg.get_string()] = msg.get_string() - - def _pack(self, msg): - self._flags = 0 - if self.st_size is not None: - self._flags |= self.FLAG_SIZE - if (self.st_uid is not None) and (self.st_gid is not None): - self._flags |= self.FLAG_UIDGID - if self.st_mode is not None: - self._flags |= self.FLAG_PERMISSIONS - if (self.st_atime is not None) and (self.st_mtime is not None): - self._flags |= self.FLAG_AMTIME - if len(self.attr) > 0: - self._flags |= self.FLAG_EXTENDED - msg.add_int(self._flags) - if self._flags & self.FLAG_SIZE: - msg.add_int64(self.st_size) - if self._flags & self.FLAG_UIDGID: - msg.add_int(self.st_uid) - msg.add_int(self.st_gid) - if self._flags & self.FLAG_PERMISSIONS: - msg.add_int(self.st_mode) - if self._flags & self.FLAG_AMTIME: - # throw away any fractional seconds - msg.add_int(long(self.st_atime)) - msg.add_int(long(self.st_mtime)) - if self._flags & self.FLAG_EXTENDED: - msg.add_int(len(self.attr)) - for key, val in self.attr.iteritems(): - msg.add_string(key) - msg.add_string(val) - return - - def _debug_str(self): - out = '[ ' - if self.st_size is not None: - out += 'size=%d ' % self.st_size - if (self.st_uid is not None) and (self.st_gid is not None): - out += 'uid=%d gid=%d ' % (self.st_uid, self.st_gid) - if self.st_mode is not None: - out += 'mode=' + oct(self.st_mode) + ' ' - if (self.st_atime is not None) and (self.st_mtime is not None): - out += 'atime=%d mtime=%d ' % (self.st_atime, self.st_mtime) - for k, v in self.attr.iteritems(): - out += '"%s"=%r ' % (str(k), v) - out += ']' - return out - - def _rwx(n, suid, sticky=False): - if suid: - suid = 2 - out = '-r'[n >> 2] + '-w'[(n >> 1) & 1] - if sticky: - out += '-xTt'[suid + (n & 1)] - else: - out += '-xSs'[suid + (n & 1)] - return out - _rwx = staticmethod(_rwx) - - def __str__(self): - "create a unix-style long description of the file (like ls -l)" - if self.st_mode is not None: - kind = stat.S_IFMT(self.st_mode) - if kind == stat.S_IFIFO: - ks = 'p' - elif kind == stat.S_IFCHR: - ks = 'c' - elif kind == stat.S_IFDIR: - ks = 'd' - elif kind == stat.S_IFBLK: - ks = 'b' - elif kind == stat.S_IFREG: - ks = '-' - elif kind == stat.S_IFLNK: - ks = 'l' - elif kind == stat.S_IFSOCK: - ks = 's' - else: - ks = '?' - ks += self._rwx((self.st_mode & 0700) >> 6, self.st_mode & stat.S_ISUID) - ks += self._rwx((self.st_mode & 070) >> 3, self.st_mode & stat.S_ISGID) - ks += self._rwx(self.st_mode & 7, self.st_mode & stat.S_ISVTX, True) - else: - ks = '?---------' - # compute display date - if (self.st_mtime is None) or (self.st_mtime == 0xffffffffL): - # shouldn't really happen - datestr = '(unknown date)' - else: - if abs(time.time() - self.st_mtime) > 15552000: - # (15552000 = 6 months) - datestr = time.strftime('%d %b %Y', time.localtime(self.st_mtime)) - else: - datestr = time.strftime('%d %b %H:%M', time.localtime(self.st_mtime)) - filename = getattr(self, 'filename', '?') - - # not all servers support uid/gid - uid = self.st_uid - gid = self.st_gid - if uid is None: - uid = 0 - if gid is None: - gid = 0 - - return '%s 1 %-8d %-8d %8d %-12s %s' % (ks, uid, gid, self.st_size, datestr, filename) - diff --git a/contrib/site-packages/paramiko/sftp_client.py b/contrib/site-packages/paramiko/sftp_client.py deleted file mode 100644 index d9215743..00000000 --- a/contrib/site-packages/paramiko/sftp_client.py +++ /dev/null @@ -1,787 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -Client-mode SFTP support. -""" - -from binascii import hexlify -import errno -import os -import stat -import threading -import time -import weakref - -from paramiko.sftp import * -from paramiko.sftp_attr import SFTPAttributes -from paramiko.ssh_exception import SSHException -from paramiko.sftp_file import SFTPFile - - -def _to_unicode(s): - """ - decode a string as ascii or utf8 if possible (as required by the sftp - protocol). if neither works, just return a byte string because the server - probably doesn't know the filename's encoding. - """ - try: - return s.encode('ascii') - except UnicodeError: - try: - return s.decode('utf-8') - except UnicodeError: - return s - - -class SFTPClient (BaseSFTP): - """ - SFTP client object. C{SFTPClient} is used to open an sftp session across - an open ssh L{Transport} and do remote file operations. - """ - - def __init__(self, sock): - """ - Create an SFTP client from an existing L{Channel}. The channel - should already have requested the C{"sftp"} subsystem. - - An alternate way to create an SFTP client context is by using - L{from_transport}. - - @param sock: an open L{Channel} using the C{"sftp"} subsystem - @type sock: L{Channel} - - @raise SSHException: if there's an exception while negotiating - sftp - """ - BaseSFTP.__init__(self) - self.sock = sock - self.ultra_debug = False - self.request_number = 1 - # lock for request_number - self._lock = threading.Lock() - self._cwd = None - # request # -> SFTPFile - self._expecting = weakref.WeakValueDictionary() - if type(sock) is Channel: - # override default logger - transport = self.sock.get_transport() - self.logger = util.get_logger(transport.get_log_channel() + '.sftp') - self.ultra_debug = transport.get_hexdump() - try: - server_version = self._send_version() - except EOFError, x: - raise SSHException('EOF during negotiation') - self._log(INFO, 'Opened sftp connection (server version %d)' % server_version) - - def from_transport(cls, t): - """ - Create an SFTP client channel from an open L{Transport}. - - @param t: an open L{Transport} which is already authenticated - @type t: L{Transport} - @return: a new L{SFTPClient} object, referring to an sftp session - (channel) across the transport - @rtype: L{SFTPClient} - """ - chan = t.open_session() - if chan is None: - return None - chan.invoke_subsystem('sftp') - return cls(chan) - from_transport = classmethod(from_transport) - - def _log(self, level, msg, *args): - if isinstance(msg, list): - for m in msg: - super(SFTPClient, self)._log(level, "[chan %s] " + m, *([ self.sock.get_name() ] + list(args))) - else: - super(SFTPClient, self)._log(level, "[chan %s] " + msg, *([ self.sock.get_name() ] + list(args))) - - def close(self): - """ - Close the SFTP session and its underlying channel. - - @since: 1.4 - """ - self._log(INFO, 'sftp session closed.') - self.sock.close() - - def get_channel(self): - """ - Return the underlying L{Channel} object for this SFTP session. This - might be useful for doing things like setting a timeout on the channel. - - @return: the SSH channel - @rtype: L{Channel} - - @since: 1.7.1 - """ - return self.sock - - def listdir(self, path='.'): - """ - Return a list containing the names of the entries in the given C{path}. - The list is in arbitrary order. It does not include the special - entries C{'.'} and C{'..'} even if they are present in the folder. - This method is meant to mirror C{os.listdir} as closely as possible. - For a list of full L{SFTPAttributes} objects, see L{listdir_attr}. - - @param path: path to list (defaults to C{'.'}) - @type path: str - @return: list of filenames - @rtype: list of str - """ - return [f.filename for f in self.listdir_attr(path)] - - def listdir_attr(self, path='.'): - """ - Return a list containing L{SFTPAttributes} objects corresponding to - files in the given C{path}. The list is in arbitrary order. It does - not include the special entries C{'.'} and C{'..'} even if they are - present in the folder. - - The returned L{SFTPAttributes} objects will each have an additional - field: C{longname}, which may contain a formatted string of the file's - attributes, in unix format. The content of this string will probably - depend on the SFTP server implementation. - - @param path: path to list (defaults to C{'.'}) - @type path: str - @return: list of attributes - @rtype: list of L{SFTPAttributes} - - @since: 1.2 - """ - path = self._adjust_cwd(path) - self._log(DEBUG, 'listdir(%r)' % path) - t, msg = self._request(CMD_OPENDIR, path) - if t != CMD_HANDLE: - raise SFTPError('Expected handle') - handle = msg.get_string() - filelist = [] - while True: - try: - t, msg = self._request(CMD_READDIR, handle) - except EOFError, e: - # done with handle - break - if t != CMD_NAME: - raise SFTPError('Expected name response') - count = msg.get_int() - for i in range(count): - filename = _to_unicode(msg.get_string()) - longname = _to_unicode(msg.get_string()) - attr = SFTPAttributes._from_msg(msg, filename, longname) - if (filename != '.') and (filename != '..'): - filelist.append(attr) - self._request(CMD_CLOSE, handle) - return filelist - - def open(self, filename, mode='r', bufsize=-1): - """ - Open a file on the remote server. The arguments are the same as for - python's built-in C{file} (aka C{open}). A file-like object is - returned, which closely mimics the behavior of a normal python file - object, including the ability to be used as a context manager. - - The mode indicates how the file is to be opened: C{'r'} for reading, - C{'w'} for writing (truncating an existing file), C{'a'} for appending, - C{'r+'} for reading/writing, C{'w+'} for reading/writing (truncating an - existing file), C{'a+'} for reading/appending. The python C{'b'} flag - is ignored, since SSH treats all files as binary. The C{'U'} flag is - supported in a compatible way. - - Since 1.5.2, an C{'x'} flag indicates that the operation should only - succeed if the file was created and did not previously exist. This has - no direct mapping to python's file flags, but is commonly known as the - C{O_EXCL} flag in posix. - - The file will be buffered in standard python style by default, but - can be altered with the C{bufsize} parameter. C{0} turns off - buffering, C{1} uses line buffering, and any number greater than 1 - (C{>1}) uses that specific buffer size. - - @param filename: name of the file to open - @type filename: str - @param mode: mode (python-style) to open in - @type mode: str - @param bufsize: desired buffering (-1 = default buffer size) - @type bufsize: int - @return: a file object representing the open file - @rtype: SFTPFile - - @raise IOError: if the file could not be opened. - """ - filename = self._adjust_cwd(filename) - self._log(DEBUG, 'open(%r, %r)' % (filename, mode)) - imode = 0 - if ('r' in mode) or ('+' in mode): - imode |= SFTP_FLAG_READ - if ('w' in mode) or ('+' in mode) or ('a' in mode): - imode |= SFTP_FLAG_WRITE - if ('w' in mode): - imode |= SFTP_FLAG_CREATE | SFTP_FLAG_TRUNC - if ('a' in mode): - imode |= SFTP_FLAG_CREATE | SFTP_FLAG_APPEND - if ('x' in mode): - imode |= SFTP_FLAG_CREATE | SFTP_FLAG_EXCL - attrblock = SFTPAttributes() - t, msg = self._request(CMD_OPEN, filename, imode, attrblock) - if t != CMD_HANDLE: - raise SFTPError('Expected handle') - handle = msg.get_string() - self._log(DEBUG, 'open(%r, %r) -> %s' % (filename, mode, hexlify(handle))) - return SFTPFile(self, handle, mode, bufsize) - - # python continues to vacillate about "open" vs "file"... - file = open - - def remove(self, path): - """ - Remove the file at the given path. This only works on files; for - removing folders (directories), use L{rmdir}. - - @param path: path (absolute or relative) of the file to remove - @type path: str - - @raise IOError: if the path refers to a folder (directory) - """ - path = self._adjust_cwd(path) - self._log(DEBUG, 'remove(%r)' % path) - self._request(CMD_REMOVE, path) - - unlink = remove - - def rename(self, oldpath, newpath): - """ - Rename a file or folder from C{oldpath} to C{newpath}. - - @param oldpath: existing name of the file or folder - @type oldpath: str - @param newpath: new name for the file or folder - @type newpath: str - - @raise IOError: if C{newpath} is a folder, or something else goes - wrong - """ - oldpath = self._adjust_cwd(oldpath) - newpath = self._adjust_cwd(newpath) - self._log(DEBUG, 'rename(%r, %r)' % (oldpath, newpath)) - self._request(CMD_RENAME, oldpath, newpath) - - def mkdir(self, path, mode=0777): - """ - Create a folder (directory) named C{path} with numeric mode C{mode}. - The default mode is 0777 (octal). On some systems, mode is ignored. - Where it is used, the current umask value is first masked out. - - @param path: name of the folder to create - @type path: str - @param mode: permissions (posix-style) for the newly-created folder - @type mode: int - """ - path = self._adjust_cwd(path) - self._log(DEBUG, 'mkdir(%r, %r)' % (path, mode)) - attr = SFTPAttributes() - attr.st_mode = mode - self._request(CMD_MKDIR, path, attr) - - def rmdir(self, path): - """ - Remove the folder named C{path}. - - @param path: name of the folder to remove - @type path: str - """ - path = self._adjust_cwd(path) - self._log(DEBUG, 'rmdir(%r)' % path) - self._request(CMD_RMDIR, path) - - def stat(self, path): - """ - Retrieve information about a file on the remote system. The return - value is an object whose attributes correspond to the attributes of - python's C{stat} structure as returned by C{os.stat}, except that it - contains fewer fields. An SFTP server may return as much or as little - info as it wants, so the results may vary from server to server. - - Unlike a python C{stat} object, the result may not be accessed as a - tuple. This is mostly due to the author's slack factor. - - The fields supported are: C{st_mode}, C{st_size}, C{st_uid}, C{st_gid}, - C{st_atime}, and C{st_mtime}. - - @param path: the filename to stat - @type path: str - @return: an object containing attributes about the given file - @rtype: SFTPAttributes - """ - path = self._adjust_cwd(path) - self._log(DEBUG, 'stat(%r)' % path) - t, msg = self._request(CMD_STAT, path) - if t != CMD_ATTRS: - raise SFTPError('Expected attributes') - return SFTPAttributes._from_msg(msg) - - def lstat(self, path): - """ - Retrieve information about a file on the remote system, without - following symbolic links (shortcuts). This otherwise behaves exactly - the same as L{stat}. - - @param path: the filename to stat - @type path: str - @return: an object containing attributes about the given file - @rtype: SFTPAttributes - """ - path = self._adjust_cwd(path) - self._log(DEBUG, 'lstat(%r)' % path) - t, msg = self._request(CMD_LSTAT, path) - if t != CMD_ATTRS: - raise SFTPError('Expected attributes') - return SFTPAttributes._from_msg(msg) - - def symlink(self, source, dest): - """ - Create a symbolic link (shortcut) of the C{source} path at - C{destination}. - - @param source: path of the original file - @type source: str - @param dest: path of the newly created symlink - @type dest: str - """ - dest = self._adjust_cwd(dest) - self._log(DEBUG, 'symlink(%r, %r)' % (source, dest)) - if type(source) is unicode: - source = source.encode('utf-8') - self._request(CMD_SYMLINK, source, dest) - - def chmod(self, path, mode): - """ - Change the mode (permissions) of a file. The permissions are - unix-style and identical to those used by python's C{os.chmod} - function. - - @param path: path of the file to change the permissions of - @type path: str - @param mode: new permissions - @type mode: int - """ - path = self._adjust_cwd(path) - self._log(DEBUG, 'chmod(%r, %r)' % (path, mode)) - attr = SFTPAttributes() - attr.st_mode = mode - self._request(CMD_SETSTAT, path, attr) - - def chown(self, path, uid, gid): - """ - Change the owner (C{uid}) and group (C{gid}) of a file. As with - python's C{os.chown} function, you must pass both arguments, so if you - only want to change one, use L{stat} first to retrieve the current - owner and group. - - @param path: path of the file to change the owner and group of - @type path: str - @param uid: new owner's uid - @type uid: int - @param gid: new group id - @type gid: int - """ - path = self._adjust_cwd(path) - self._log(DEBUG, 'chown(%r, %r, %r)' % (path, uid, gid)) - attr = SFTPAttributes() - attr.st_uid, attr.st_gid = uid, gid - self._request(CMD_SETSTAT, path, attr) - - def utime(self, path, times): - """ - Set the access and modified times of the file specified by C{path}. If - C{times} is C{None}, then the file's access and modified times are set - to the current time. Otherwise, C{times} must be a 2-tuple of numbers, - of the form C{(atime, mtime)}, which is used to set the access and - modified times, respectively. This bizarre API is mimicked from python - for the sake of consistency -- I apologize. - - @param path: path of the file to modify - @type path: str - @param times: C{None} or a tuple of (access time, modified time) in - standard internet epoch time (seconds since 01 January 1970 GMT) - @type times: tuple(int) - """ - path = self._adjust_cwd(path) - if times is None: - times = (time.time(), time.time()) - self._log(DEBUG, 'utime(%r, %r)' % (path, times)) - attr = SFTPAttributes() - attr.st_atime, attr.st_mtime = times - self._request(CMD_SETSTAT, path, attr) - - def truncate(self, path, size): - """ - Change the size of the file specified by C{path}. This usually extends - or shrinks the size of the file, just like the C{truncate()} method on - python file objects. - - @param path: path of the file to modify - @type path: str - @param size: the new size of the file - @type size: int or long - """ - path = self._adjust_cwd(path) - self._log(DEBUG, 'truncate(%r, %r)' % (path, size)) - attr = SFTPAttributes() - attr.st_size = size - self._request(CMD_SETSTAT, path, attr) - - def readlink(self, path): - """ - Return the target of a symbolic link (shortcut). You can use - L{symlink} to create these. The result may be either an absolute or - relative pathname. - - @param path: path of the symbolic link file - @type path: str - @return: target path - @rtype: str - """ - path = self._adjust_cwd(path) - self._log(DEBUG, 'readlink(%r)' % path) - t, msg = self._request(CMD_READLINK, path) - if t != CMD_NAME: - raise SFTPError('Expected name response') - count = msg.get_int() - if count == 0: - return None - if count != 1: - raise SFTPError('Readlink returned %d results' % count) - return _to_unicode(msg.get_string()) - - def normalize(self, path): - """ - Return the normalized path (on the server) of a given path. This - can be used to quickly resolve symbolic links or determine what the - server is considering to be the "current folder" (by passing C{'.'} - as C{path}). - - @param path: path to be normalized - @type path: str - @return: normalized form of the given path - @rtype: str - - @raise IOError: if the path can't be resolved on the server - """ - path = self._adjust_cwd(path) - self._log(DEBUG, 'normalize(%r)' % path) - t, msg = self._request(CMD_REALPATH, path) - if t != CMD_NAME: - raise SFTPError('Expected name response') - count = msg.get_int() - if count != 1: - raise SFTPError('Realpath returned %d results' % count) - return _to_unicode(msg.get_string()) - - def chdir(self, path): - """ - Change the "current directory" of this SFTP session. Since SFTP - doesn't really have the concept of a current working directory, this - is emulated by paramiko. Once you use this method to set a working - directory, all operations on this SFTPClient object will be relative - to that path. You can pass in C{None} to stop using a current working - directory. - - @param path: new current working directory - @type path: str - - @raise IOError: if the requested path doesn't exist on the server - - @since: 1.4 - """ - if path is None: - self._cwd = None - return - if not stat.S_ISDIR(self.stat(path).st_mode): - raise SFTPError(errno.ENOTDIR, "%s: %s" % (os.strerror(errno.ENOTDIR), path)) - self._cwd = self.normalize(path).encode('utf-8') - - def getcwd(self): - """ - Return the "current working directory" for this SFTP session, as - emulated by paramiko. If no directory has been set with L{chdir}, - this method will return C{None}. - - @return: the current working directory on the server, or C{None} - @rtype: str - - @since: 1.4 - """ - return self._cwd - - def putfo(self, fl, remotepath, file_size=0, callback=None, confirm=True): - """ - Copy the contents of an open file object (C{fl}) to the SFTP server as - C{remotepath}. Any exception raised by operations will be passed through. - - The SFTP operations use pipelining for speed. - - @param fl: opened file or file-like object to copy - @type localpath: object - @param remotepath: the destination path on the SFTP server - @type remotepath: str - @param file_size: optional size parameter passed to callback. If none is - specified, size defaults to 0 - @type file_size: int - @param callback: optional callback function that accepts the bytes - transferred so far and the total bytes to be transferred - (since 1.7.4) - @type callback: function(int, int) - @param confirm: whether to do a stat() on the file afterwards to - confirm the file size (since 1.7.7) - @type confirm: bool - - @return: an object containing attributes about the given file - (since 1.7.4) - @rtype: SFTPAttributes - - @since: 1.4 - """ - fr = self.file(remotepath, 'wb') - fr.set_pipelined(True) - size = 0 - try: - while True: - data = fl.read(32768) - fr.write(data) - size += len(data) - if callback is not None: - callback(size, file_size) - if len(data) == 0: - break - finally: - fr.close() - if confirm: - s = self.stat(remotepath) - if s.st_size != size: - raise IOError('size mismatch in put! %d != %d' % (s.st_size, size)) - else: - s = SFTPAttributes() - return s - - def put(self, localpath, remotepath, callback=None, confirm=True): - """ - Copy a local file (C{localpath}) to the SFTP server as C{remotepath}. - Any exception raised by operations will be passed through. This - method is primarily provided as a convenience. - - The SFTP operations use pipelining for speed. - - @param localpath: the local file to copy - @type localpath: str - @param remotepath: the destination path on the SFTP server - @type remotepath: str - @param callback: optional callback function that accepts the bytes - transferred so far and the total bytes to be transferred - (since 1.7.4) - @type callback: function(int, int) - @param confirm: whether to do a stat() on the file afterwards to - confirm the file size (since 1.7.7) - @type confirm: bool - - @return: an object containing attributes about the given file - (since 1.7.4) - @rtype: SFTPAttributes - - @since: 1.4 - """ - file_size = os.stat(localpath).st_size - fl = file(localpath, 'rb') - try: - return self.putfo(fl, remotepath, os.stat(localpath).st_size, callback, confirm) - finally: - fl.close() - - def getfo(self, remotepath, fl, callback=None): - """ - Copy a remote file (C{remotepath}) from the SFTP server and write to - an open file or file-like object, C{fl}. Any exception raised by - operations will be passed through. This method is primarily provided - as a convenience. - - @param remotepath: opened file or file-like object to copy to - @type remotepath: object - @param fl: the destination path on the local host or open file - object - @type localpath: str - @param callback: optional callback function that accepts the bytes - transferred so far and the total bytes to be transferred - (since 1.7.4) - @type callback: function(int, int) - @return: the number of bytes written to the opened file object - - @since: 1.4 - """ - fr = self.file(remotepath, 'rb') - file_size = self.stat(remotepath).st_size - fr.prefetch() - try: - size = 0 - while True: - data = fr.read(32768) - fl.write(data) - size += len(data) - if callback is not None: - callback(size, file_size) - if len(data) == 0: - break - finally: - fr.close() - return size - - def get(self, remotepath, localpath, callback=None): - """ - Copy a remote file (C{remotepath}) from the SFTP server to the local - host as C{localpath}. Any exception raised by operations will be - passed through. This method is primarily provided as a convenience. - - @param remotepath: the remote file to copy - @type remotepath: str - @param localpath: the destination path on the local host - @type localpath: str - @param callback: optional callback function that accepts the bytes - transferred so far and the total bytes to be transferred - (since 1.7.4) - @type callback: function(int, int) - - @since: 1.4 - """ - file_size = self.stat(remotepath).st_size - fl = file(localpath, 'wb') - try: - size = self.getfo(remotepath, fl, callback) - finally: - fl.close() - s = os.stat(localpath) - if s.st_size != size: - raise IOError('size mismatch in get! %d != %d' % (s.st_size, size)) - - - ### internals... - - - def _request(self, t, *arg): - num = self._async_request(type(None), t, *arg) - return self._read_response(num) - - def _async_request(self, fileobj, t, *arg): - # this method may be called from other threads (prefetch) - self._lock.acquire() - try: - msg = Message() - msg.add_int(self.request_number) - for item in arg: - if isinstance(item, int): - msg.add_int(item) - elif isinstance(item, long): - msg.add_int64(item) - elif isinstance(item, str): - msg.add_string(item) - elif isinstance(item, SFTPAttributes): - item._pack(msg) - else: - raise Exception('unknown type for %r type %r' % (item, type(item))) - num = self.request_number - self._expecting[num] = fileobj - self._send_packet(t, str(msg)) - self.request_number += 1 - finally: - self._lock.release() - return num - - def _read_response(self, waitfor=None): - while True: - try: - t, data = self._read_packet() - except EOFError, e: - raise SSHException('Server connection dropped: %s' % (str(e),)) - msg = Message(data) - num = msg.get_int() - if num not in self._expecting: - # might be response for a file that was closed before responses came back - self._log(DEBUG, 'Unexpected response #%d' % (num,)) - if waitfor is None: - # just doing a single check - break - continue - fileobj = self._expecting[num] - del self._expecting[num] - if num == waitfor: - # synchronous - if t == CMD_STATUS: - self._convert_status(msg) - return t, msg - if fileobj is not type(None): - fileobj._async_response(t, msg) - if waitfor is None: - # just doing a single check - break - return (None, None) - - def _finish_responses(self, fileobj): - while fileobj in self._expecting.values(): - self._read_response() - fileobj._check_exception() - - def _convert_status(self, msg): - """ - Raises EOFError or IOError on error status; otherwise does nothing. - """ - code = msg.get_int() - text = msg.get_string() - if code == SFTP_OK: - return - elif code == SFTP_EOF: - raise EOFError(text) - elif code == SFTP_NO_SUCH_FILE: - # clever idea from john a. meinel: map the error codes to errno - raise IOError(errno.ENOENT, text) - elif code == SFTP_PERMISSION_DENIED: - raise IOError(errno.EACCES, text) - else: - raise IOError(text) - - def _adjust_cwd(self, path): - """ - Return an adjusted path if we're emulating a "current working - directory" for the server. - """ - if type(path) is unicode: - path = path.encode('utf-8') - if self._cwd is None: - return path - if (len(path) > 0) and (path[0] == '/'): - # absolute path - return path - if self._cwd == '/': - return self._cwd + path - return self._cwd + '/' + path - - -class SFTP (SFTPClient): - "an alias for L{SFTPClient} for backwards compatability" - pass diff --git a/contrib/site-packages/paramiko/sftp_file.py b/contrib/site-packages/paramiko/sftp_file.py deleted file mode 100644 index 4ec936d0..00000000 --- a/contrib/site-packages/paramiko/sftp_file.py +++ /dev/null @@ -1,489 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -L{SFTPFile} -""" - -from binascii import hexlify -from collections import deque -import socket -import threading -import time - -from paramiko.common import * -from paramiko.sftp import * -from paramiko.file import BufferedFile -from paramiko.sftp_attr import SFTPAttributes - - -class SFTPFile (BufferedFile): - """ - Proxy object for a file on the remote server, in client mode SFTP. - - Instances of this class may be used as context managers in the same way - that built-in Python file objects are. - """ - - # Some sftp servers will choke if you send read/write requests larger than - # this size. - MAX_REQUEST_SIZE = 32768 - - def __init__(self, sftp, handle, mode='r', bufsize=-1): - BufferedFile.__init__(self) - self.sftp = sftp - self.handle = handle - BufferedFile._set_mode(self, mode, bufsize) - self.pipelined = False - self._prefetching = False - self._prefetch_done = False - self._prefetch_data = {} - self._prefetch_reads = [] - self._saved_exception = None - self._reqs = deque() - - def __del__(self): - self._close(async=True) - - def close(self): - self._close(async=False) - - def _close(self, async=False): - # We allow double-close without signaling an error, because real - # Python file objects do. However, we must protect against actually - # sending multiple CMD_CLOSE packets, because after we close our - # handle, the same handle may be re-allocated by the server, and we - # may end up mysteriously closing some random other file. (This is - # especially important because we unconditionally call close() from - # __del__.) - if self._closed: - return - self.sftp._log(DEBUG, 'close(%s)' % hexlify(self.handle)) - if self.pipelined: - self.sftp._finish_responses(self) - BufferedFile.close(self) - try: - if async: - # GC'd file handle could be called from an arbitrary thread -- don't wait for a response - self.sftp._async_request(type(None), CMD_CLOSE, self.handle) - else: - self.sftp._request(CMD_CLOSE, self.handle) - except EOFError: - # may have outlived the Transport connection - pass - except (IOError, socket.error): - # may have outlived the Transport connection - pass - - def _data_in_prefetch_requests(self, offset, size): - k = [i for i in self._prefetch_reads if i[0] <= offset] - if len(k) == 0: - return False - k.sort(lambda x, y: cmp(x[0], y[0])) - buf_offset, buf_size = k[-1] - if buf_offset + buf_size <= offset: - # prefetch request ends before this one begins - return False - if buf_offset + buf_size >= offset + size: - # inclusive - return True - # well, we have part of the request. see if another chunk has the rest. - return self._data_in_prefetch_requests(buf_offset + buf_size, offset + size - buf_offset - buf_size) - - def _data_in_prefetch_buffers(self, offset): - """ - if a block of data is present in the prefetch buffers, at the given - offset, return the offset of the relevant prefetch buffer. otherwise, - return None. this guarantees nothing about the number of bytes - collected in the prefetch buffer so far. - """ - k = [i for i in self._prefetch_data.keys() if i <= offset] - if len(k) == 0: - return None - index = max(k) - buf_offset = offset - index - if buf_offset >= len(self._prefetch_data[index]): - # it's not here - return None - return index - - def _read_prefetch(self, size): - """ - read data out of the prefetch buffer, if possible. if the data isn't - in the buffer, return None. otherwise, behaves like a normal read. - """ - # while not closed, and haven't fetched past the current position, and haven't reached EOF... - while True: - offset = self._data_in_prefetch_buffers(self._realpos) - if offset is not None: - break - if self._prefetch_done or self._closed: - break - self.sftp._read_response() - self._check_exception() - if offset is None: - self._prefetching = False - return None - prefetch = self._prefetch_data[offset] - del self._prefetch_data[offset] - - buf_offset = self._realpos - offset - if buf_offset > 0: - self._prefetch_data[offset] = prefetch[:buf_offset] - prefetch = prefetch[buf_offset:] - if size < len(prefetch): - self._prefetch_data[self._realpos + size] = prefetch[size:] - prefetch = prefetch[:size] - return prefetch - - def _read(self, size): - size = min(size, self.MAX_REQUEST_SIZE) - if self._prefetching: - data = self._read_prefetch(size) - if data is not None: - return data - t, msg = self.sftp._request(CMD_READ, self.handle, long(self._realpos), int(size)) - if t != CMD_DATA: - raise SFTPError('Expected data') - return msg.get_string() - - def _write(self, data): - # may write less than requested if it would exceed max packet size - chunk = min(len(data), self.MAX_REQUEST_SIZE) - self._reqs.append(self.sftp._async_request(type(None), CMD_WRITE, self.handle, long(self._realpos), str(data[:chunk]))) - if not self.pipelined or (len(self._reqs) > 100 and self.sftp.sock.recv_ready()): - while len(self._reqs): - req = self._reqs.popleft() - t, msg = self.sftp._read_response(req) - if t != CMD_STATUS: - raise SFTPError('Expected status') - # convert_status already called - return chunk - - def settimeout(self, timeout): - """ - Set a timeout on read/write operations on the underlying socket or - ssh L{Channel}. - - @see: L{Channel.settimeout} - @param timeout: seconds to wait for a pending read/write operation - before raising C{socket.timeout}, or C{None} for no timeout - @type timeout: float - """ - self.sftp.sock.settimeout(timeout) - - def gettimeout(self): - """ - Returns the timeout in seconds (as a float) associated with the socket - or ssh L{Channel} used for this file. - - @see: L{Channel.gettimeout} - @rtype: float - """ - return self.sftp.sock.gettimeout() - - def setblocking(self, blocking): - """ - Set blocking or non-blocking mode on the underiying socket or ssh - L{Channel}. - - @see: L{Channel.setblocking} - @param blocking: 0 to set non-blocking mode; non-0 to set blocking - mode. - @type blocking: int - """ - self.sftp.sock.setblocking(blocking) - - def seek(self, offset, whence=0): - self.flush() - if whence == self.SEEK_SET: - self._realpos = self._pos = offset - elif whence == self.SEEK_CUR: - self._pos += offset - self._realpos = self._pos - else: - self._realpos = self._pos = self._get_size() + offset - self._rbuffer = '' - - def stat(self): - """ - Retrieve information about this file from the remote system. This is - exactly like L{SFTP.stat}, except that it operates on an already-open - file. - - @return: an object containing attributes about this file. - @rtype: SFTPAttributes - """ - t, msg = self.sftp._request(CMD_FSTAT, self.handle) - if t != CMD_ATTRS: - raise SFTPError('Expected attributes') - return SFTPAttributes._from_msg(msg) - - def chmod(self, mode): - """ - Change the mode (permissions) of this file. The permissions are - unix-style and identical to those used by python's C{os.chmod} - function. - - @param mode: new permissions - @type mode: int - """ - self.sftp._log(DEBUG, 'chmod(%s, %r)' % (hexlify(self.handle), mode)) - attr = SFTPAttributes() - attr.st_mode = mode - self.sftp._request(CMD_FSETSTAT, self.handle, attr) - - def chown(self, uid, gid): - """ - Change the owner (C{uid}) and group (C{gid}) of this file. As with - python's C{os.chown} function, you must pass both arguments, so if you - only want to change one, use L{stat} first to retrieve the current - owner and group. - - @param uid: new owner's uid - @type uid: int - @param gid: new group id - @type gid: int - """ - self.sftp._log(DEBUG, 'chown(%s, %r, %r)' % (hexlify(self.handle), uid, gid)) - attr = SFTPAttributes() - attr.st_uid, attr.st_gid = uid, gid - self.sftp._request(CMD_FSETSTAT, self.handle, attr) - - def utime(self, times): - """ - Set the access and modified times of this file. If - C{times} is C{None}, then the file's access and modified times are set - to the current time. Otherwise, C{times} must be a 2-tuple of numbers, - of the form C{(atime, mtime)}, which is used to set the access and - modified times, respectively. This bizarre API is mimicked from python - for the sake of consistency -- I apologize. - - @param times: C{None} or a tuple of (access time, modified time) in - standard internet epoch time (seconds since 01 January 1970 GMT) - @type times: tuple(int) - """ - if times is None: - times = (time.time(), time.time()) - self.sftp._log(DEBUG, 'utime(%s, %r)' % (hexlify(self.handle), times)) - attr = SFTPAttributes() - attr.st_atime, attr.st_mtime = times - self.sftp._request(CMD_FSETSTAT, self.handle, attr) - - def truncate(self, size): - """ - Change the size of this file. This usually extends - or shrinks the size of the file, just like the C{truncate()} method on - python file objects. - - @param size: the new size of the file - @type size: int or long - """ - self.sftp._log(DEBUG, 'truncate(%s, %r)' % (hexlify(self.handle), size)) - attr = SFTPAttributes() - attr.st_size = size - self.sftp._request(CMD_FSETSTAT, self.handle, attr) - - def check(self, hash_algorithm, offset=0, length=0, block_size=0): - """ - Ask the server for a hash of a section of this file. This can be used - to verify a successful upload or download, or for various rsync-like - operations. - - The file is hashed from C{offset}, for C{length} bytes. If C{length} - is 0, the remainder of the file is hashed. Thus, if both C{offset} - and C{length} are zero, the entire file is hashed. - - Normally, C{block_size} will be 0 (the default), and this method will - return a byte string representing the requested hash (for example, a - string of length 16 for MD5, or 20 for SHA-1). If a non-zero - C{block_size} is given, each chunk of the file (from C{offset} to - C{offset + length}) of C{block_size} bytes is computed as a separate - hash. The hash results are all concatenated and returned as a single - string. - - For example, C{check('sha1', 0, 1024, 512)} will return a string of - length 40. The first 20 bytes will be the SHA-1 of the first 512 bytes - of the file, and the last 20 bytes will be the SHA-1 of the next 512 - bytes. - - @param hash_algorithm: the name of the hash algorithm to use (normally - C{"sha1"} or C{"md5"}) - @type hash_algorithm: str - @param offset: offset into the file to begin hashing (0 means to start - from the beginning) - @type offset: int or long - @param length: number of bytes to hash (0 means continue to the end of - the file) - @type length: int or long - @param block_size: number of bytes to hash per result (must not be less - than 256; 0 means to compute only one hash of the entire segment) - @type block_size: int - @return: string of bytes representing the hash of each block, - concatenated together - @rtype: str - - @note: Many (most?) servers don't support this extension yet. - - @raise IOError: if the server doesn't support the "check-file" - extension, or possibly doesn't support the hash algorithm - requested - - @since: 1.4 - """ - t, msg = self.sftp._request(CMD_EXTENDED, 'check-file', self.handle, - hash_algorithm, long(offset), long(length), block_size) - ext = msg.get_string() - alg = msg.get_string() - data = msg.get_remainder() - return data - - def set_pipelined(self, pipelined=True): - """ - Turn on/off the pipelining of write operations to this file. When - pipelining is on, paramiko won't wait for the server response after - each write operation. Instead, they're collected as they come in. - At the first non-write operation (including L{close}), all remaining - server responses are collected. This means that if there was an error - with one of your later writes, an exception might be thrown from - within L{close} instead of L{write}. - - By default, files are I{not} pipelined. - - @param pipelined: C{True} if pipelining should be turned on for this - file; C{False} otherwise - @type pipelined: bool - - @since: 1.5 - """ - self.pipelined = pipelined - - def prefetch(self): - """ - Pre-fetch the remaining contents of this file in anticipation of - future L{read} calls. If reading the entire file, pre-fetching can - dramatically improve the download speed by avoiding roundtrip latency. - The file's contents are incrementally buffered in a background thread. - - The prefetched data is stored in a buffer until read via the L{read} - method. Once data has been read, it's removed from the buffer. The - data may be read in a random order (using L{seek}); chunks of the - buffer that haven't been read will continue to be buffered. - - @since: 1.5.1 - """ - size = self.stat().st_size - # queue up async reads for the rest of the file - chunks = [] - n = self._realpos - while n < size: - chunk = min(self.MAX_REQUEST_SIZE, size - n) - chunks.append((n, chunk)) - n += chunk - if len(chunks) > 0: - self._start_prefetch(chunks) - - def readv(self, chunks): - """ - Read a set of blocks from the file by (offset, length). This is more - efficient than doing a series of L{seek} and L{read} calls, since the - prefetch machinery is used to retrieve all the requested blocks at - once. - - @param chunks: a list of (offset, length) tuples indicating which - sections of the file to read - @type chunks: list(tuple(long, int)) - @return: a list of blocks read, in the same order as in C{chunks} - @rtype: list(str) - - @since: 1.5.4 - """ - self.sftp._log(DEBUG, 'readv(%s, %r)' % (hexlify(self.handle), chunks)) - - read_chunks = [] - for offset, size in chunks: - # don't fetch data that's already in the prefetch buffer - if self._data_in_prefetch_buffers(offset) or self._data_in_prefetch_requests(offset, size): - continue - - # break up anything larger than the max read size - while size > 0: - chunk_size = min(size, self.MAX_REQUEST_SIZE) - read_chunks.append((offset, chunk_size)) - offset += chunk_size - size -= chunk_size - - self._start_prefetch(read_chunks) - # now we can just devolve to a bunch of read()s :) - for x in chunks: - self.seek(x[0]) - yield self.read(x[1]) - - - ### internals... - - - def _get_size(self): - try: - return self.stat().st_size - except: - return 0 - - def _start_prefetch(self, chunks): - self._prefetching = True - self._prefetch_done = False - self._prefetch_reads.extend(chunks) - - t = threading.Thread(target=self._prefetch_thread, args=(chunks,)) - t.setDaemon(True) - t.start() - - def _prefetch_thread(self, chunks): - # do these read requests in a temporary thread because there may be - # a lot of them, so it may block. - for offset, length in chunks: - self.sftp._async_request(self, CMD_READ, self.handle, long(offset), int(length)) - - def _async_response(self, t, msg): - if t == CMD_STATUS: - # save exception and re-raise it on next file operation - try: - self.sftp._convert_status(msg) - except Exception, x: - self._saved_exception = x - return - if t != CMD_DATA: - raise SFTPError('Expected data') - data = msg.get_string() - offset, length = self._prefetch_reads.pop(0) - self._prefetch_data[offset] = data - if len(self._prefetch_reads) == 0: - self._prefetch_done = True - - def _check_exception(self): - "if there's a saved exception, raise & clear it" - if self._saved_exception is not None: - x = self._saved_exception - self._saved_exception = None - raise x - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.close() diff --git a/contrib/site-packages/paramiko/sftp_handle.py b/contrib/site-packages/paramiko/sftp_handle.py deleted file mode 100644 index 29d3d0d8..00000000 --- a/contrib/site-packages/paramiko/sftp_handle.py +++ /dev/null @@ -1,202 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -Abstraction of an SFTP file handle (for server mode). -""" - -import os - -from paramiko.common import * -from paramiko.sftp import * - - -class SFTPHandle (object): - """ - Abstract object representing a handle to an open file (or folder) in an - SFTP server implementation. Each handle has a string representation used - by the client to refer to the underlying file. - - Server implementations can (and should) subclass SFTPHandle to implement - features of a file handle, like L{stat} or L{chattr}. - """ - def __init__(self, flags=0): - """ - Create a new file handle representing a local file being served over - SFTP. If C{flags} is passed in, it's used to determine if the file - is open in append mode. - - @param flags: optional flags as passed to L{SFTPServerInterface.open} - @type flags: int - """ - self.__flags = flags - self.__name = None - # only for handles to folders: - self.__files = { } - self.__tell = None - - def close(self): - """ - When a client closes a file, this method is called on the handle. - Normally you would use this method to close the underlying OS level - file object(s). - - The default implementation checks for attributes on C{self} named - C{readfile} and/or C{writefile}, and if either or both are present, - their C{close()} methods are called. This means that if you are - using the default implementations of L{read} and L{write}, this - method's default implementation should be fine also. - """ - readfile = getattr(self, 'readfile', None) - if readfile is not None: - readfile.close() - writefile = getattr(self, 'writefile', None) - if writefile is not None: - writefile.close() - - def read(self, offset, length): - """ - Read up to C{length} bytes from this file, starting at position - C{offset}. The offset may be a python long, since SFTP allows it - to be 64 bits. - - If the end of the file has been reached, this method may return an - empty string to signify EOF, or it may also return L{SFTP_EOF}. - - The default implementation checks for an attribute on C{self} named - C{readfile}, and if present, performs the read operation on the python - file-like object found there. (This is meant as a time saver for the - common case where you are wrapping a python file object.) - - @param offset: position in the file to start reading from. - @type offset: int or long - @param length: number of bytes to attempt to read. - @type length: int - @return: data read from the file, or an SFTP error code. - @rtype: str - """ - readfile = getattr(self, 'readfile', None) - if readfile is None: - return SFTP_OP_UNSUPPORTED - try: - if self.__tell is None: - self.__tell = readfile.tell() - if offset != self.__tell: - readfile.seek(offset) - self.__tell = offset - data = readfile.read(length) - except IOError, e: - self.__tell = None - return SFTPServer.convert_errno(e.errno) - self.__tell += len(data) - return data - - def write(self, offset, data): - """ - Write C{data} into this file at position C{offset}. Extending the - file past its original end is expected. Unlike python's normal - C{write()} methods, this method cannot do a partial write: it must - write all of C{data} or else return an error. - - The default implementation checks for an attribute on C{self} named - C{writefile}, and if present, performs the write operation on the - python file-like object found there. The attribute is named - differently from C{readfile} to make it easy to implement read-only - (or write-only) files, but if both attributes are present, they should - refer to the same file. - - @param offset: position in the file to start reading from. - @type offset: int or long - @param data: data to write into the file. - @type data: str - @return: an SFTP error code like L{SFTP_OK}. - """ - writefile = getattr(self, 'writefile', None) - if writefile is None: - return SFTP_OP_UNSUPPORTED - try: - # in append mode, don't care about seeking - if (self.__flags & os.O_APPEND) == 0: - if self.__tell is None: - self.__tell = writefile.tell() - if offset != self.__tell: - writefile.seek(offset) - self.__tell = offset - writefile.write(data) - writefile.flush() - except IOError, e: - self.__tell = None - return SFTPServer.convert_errno(e.errno) - if self.__tell is not None: - self.__tell += len(data) - return SFTP_OK - - def stat(self): - """ - Return an L{SFTPAttributes} object referring to this open file, or an - error code. This is equivalent to L{SFTPServerInterface.stat}, except - it's called on an open file instead of a path. - - @return: an attributes object for the given file, or an SFTP error - code (like L{SFTP_PERMISSION_DENIED}). - @rtype: L{SFTPAttributes} I{or error code} - """ - return SFTP_OP_UNSUPPORTED - - def chattr(self, attr): - """ - Change the attributes of this file. The C{attr} object will contain - only those fields provided by the client in its request, so you should - check for the presence of fields before using them. - - @param attr: the attributes to change on this file. - @type attr: L{SFTPAttributes} - @return: an error code like L{SFTP_OK}. - @rtype: int - """ - return SFTP_OP_UNSUPPORTED - - - ### internals... - - - def _set_files(self, files): - """ - Used by the SFTP server code to cache a directory listing. (In - the SFTP protocol, listing a directory is a multi-stage process - requiring a temporary handle.) - """ - self.__files = files - - def _get_next_files(self): - """ - Used by the SFTP server code to retreive a cached directory - listing. - """ - fnlist = self.__files[:16] - self.__files = self.__files[16:] - return fnlist - - def _get_name(self): - return self.__name - - def _set_name(self, name): - self.__name = name - - -from paramiko.sftp_server import SFTPServer diff --git a/contrib/site-packages/paramiko/sftp_server.py b/contrib/site-packages/paramiko/sftp_server.py deleted file mode 100644 index 1833c2e8..00000000 --- a/contrib/site-packages/paramiko/sftp_server.py +++ /dev/null @@ -1,444 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -Server-mode SFTP support. -""" - -import os -import errno - -from Crypto.Hash import MD5, SHA -from paramiko.common import * -from paramiko.server import SubsystemHandler -from paramiko.sftp import * -from paramiko.sftp_si import * -from paramiko.sftp_attr import * - - -# known hash algorithms for the "check-file" extension -_hash_class = { - 'sha1': SHA, - 'md5': MD5, -} - - -class SFTPServer (BaseSFTP, SubsystemHandler): - """ - Server-side SFTP subsystem support. Since this is a L{SubsystemHandler}, - it can be (and is meant to be) set as the handler for C{"sftp"} requests. - Use L{Transport.set_subsystem_handler} to activate this class. - """ - - def __init__(self, channel, name, server, sftp_si=SFTPServerInterface, *largs, **kwargs): - """ - The constructor for SFTPServer is meant to be called from within the - L{Transport} as a subsystem handler. C{server} and any additional - parameters or keyword parameters are passed from the original call to - L{Transport.set_subsystem_handler}. - - @param channel: channel passed from the L{Transport}. - @type channel: L{Channel} - @param name: name of the requested subsystem. - @type name: str - @param server: the server object associated with this channel and - subsystem - @type server: L{ServerInterface} - @param sftp_si: a subclass of L{SFTPServerInterface} to use for handling - individual requests. - @type sftp_si: class - """ - BaseSFTP.__init__(self) - SubsystemHandler.__init__(self, channel, name, server) - transport = channel.get_transport() - self.logger = util.get_logger(transport.get_log_channel() + '.sftp') - self.ultra_debug = transport.get_hexdump() - self.next_handle = 1 - # map of handle-string to SFTPHandle for files & folders: - self.file_table = { } - self.folder_table = { } - self.server = sftp_si(server, *largs, **kwargs) - - def _log(self, level, msg): - if issubclass(type(msg), list): - for m in msg: - super(SFTPServer, self)._log(level, "[chan " + self.sock.get_name() + "] " + m) - else: - super(SFTPServer, self)._log(level, "[chan " + self.sock.get_name() + "] " + msg) - - def start_subsystem(self, name, transport, channel): - self.sock = channel - self._log(DEBUG, 'Started sftp server on channel %s' % repr(channel)) - self._send_server_version() - self.server.session_started() - while True: - try: - t, data = self._read_packet() - except EOFError: - self._log(DEBUG, 'EOF -- end of session') - return - except Exception, e: - self._log(DEBUG, 'Exception on channel: ' + str(e)) - self._log(DEBUG, util.tb_strings()) - return - msg = Message(data) - request_number = msg.get_int() - try: - self._process(t, request_number, msg) - except Exception, e: - self._log(DEBUG, 'Exception in server processing: ' + str(e)) - self._log(DEBUG, util.tb_strings()) - # send some kind of failure message, at least - try: - self._send_status(request_number, SFTP_FAILURE) - except: - pass - - def finish_subsystem(self): - self.server.session_ended() - super(SFTPServer, self).finish_subsystem() - # close any file handles that were left open (so we can return them to the OS quickly) - for f in self.file_table.itervalues(): - f.close() - for f in self.folder_table.itervalues(): - f.close() - self.file_table = {} - self.folder_table = {} - - def convert_errno(e): - """ - Convert an errno value (as from an C{OSError} or C{IOError}) into a - standard SFTP result code. This is a convenience function for trapping - exceptions in server code and returning an appropriate result. - - @param e: an errno code, as from C{OSError.errno}. - @type e: int - @return: an SFTP error code like L{SFTP_NO_SUCH_FILE}. - @rtype: int - """ - if e == errno.EACCES: - # permission denied - return SFTP_PERMISSION_DENIED - elif (e == errno.ENOENT) or (e == errno.ENOTDIR): - # no such file - return SFTP_NO_SUCH_FILE - else: - return SFTP_FAILURE - convert_errno = staticmethod(convert_errno) - - def set_file_attr(filename, attr): - """ - Change a file's attributes on the local filesystem. The contents of - C{attr} are used to change the permissions, owner, group ownership, - and/or modification & access time of the file, depending on which - attributes are present in C{attr}. - - This is meant to be a handy helper function for translating SFTP file - requests into local file operations. - - @param filename: name of the file to alter (should usually be an - absolute path). - @type filename: str - @param attr: attributes to change. - @type attr: L{SFTPAttributes} - """ - if sys.platform != 'win32': - # mode operations are meaningless on win32 - if attr._flags & attr.FLAG_PERMISSIONS: - os.chmod(filename, attr.st_mode) - if attr._flags & attr.FLAG_UIDGID: - os.chown(filename, attr.st_uid, attr.st_gid) - if attr._flags & attr.FLAG_AMTIME: - os.utime(filename, (attr.st_atime, attr.st_mtime)) - if attr._flags & attr.FLAG_SIZE: - open(filename, 'w+').truncate(attr.st_size) - set_file_attr = staticmethod(set_file_attr) - - - ### internals... - - - def _response(self, request_number, t, *arg): - msg = Message() - msg.add_int(request_number) - for item in arg: - if type(item) is int: - msg.add_int(item) - elif type(item) is long: - msg.add_int64(item) - elif type(item) is str: - msg.add_string(item) - elif type(item) is SFTPAttributes: - item._pack(msg) - else: - raise Exception('unknown type for ' + repr(item) + ' type ' + repr(type(item))) - self._send_packet(t, str(msg)) - - def _send_handle_response(self, request_number, handle, folder=False): - if not issubclass(type(handle), SFTPHandle): - # must be error code - self._send_status(request_number, handle) - return - handle._set_name('hx%d' % self.next_handle) - self.next_handle += 1 - if folder: - self.folder_table[handle._get_name()] = handle - else: - self.file_table[handle._get_name()] = handle - self._response(request_number, CMD_HANDLE, handle._get_name()) - - def _send_status(self, request_number, code, desc=None): - if desc is None: - try: - desc = SFTP_DESC[code] - except IndexError: - desc = 'Unknown' - # some clients expect a "langauge" tag at the end (but don't mind it being blank) - self._response(request_number, CMD_STATUS, code, desc, '') - - def _open_folder(self, request_number, path): - resp = self.server.list_folder(path) - if issubclass(type(resp), list): - # got an actual list of filenames in the folder - folder = SFTPHandle() - folder._set_files(resp) - self._send_handle_response(request_number, folder, True) - return - # must be an error code - self._send_status(request_number, resp) - - def _read_folder(self, request_number, folder): - flist = folder._get_next_files() - if len(flist) == 0: - self._send_status(request_number, SFTP_EOF) - return - msg = Message() - msg.add_int(request_number) - msg.add_int(len(flist)) - for attr in flist: - msg.add_string(attr.filename) - msg.add_string(str(attr)) - attr._pack(msg) - self._send_packet(CMD_NAME, str(msg)) - - def _check_file(self, request_number, msg): - # this extension actually comes from v6 protocol, but since it's an - # extension, i feel like we can reasonably support it backported. - # it's very useful for verifying uploaded files or checking for - # rsync-like differences between local and remote files. - handle = msg.get_string() - alg_list = msg.get_list() - start = msg.get_int64() - length = msg.get_int64() - block_size = msg.get_int() - if handle not in self.file_table: - self._send_status(request_number, SFTP_BAD_MESSAGE, 'Invalid handle') - return - f = self.file_table[handle] - for x in alg_list: - if x in _hash_class: - algname = x - alg = _hash_class[x] - break - else: - self._send_status(request_number, SFTP_FAILURE, 'No supported hash types found') - return - if length == 0: - st = f.stat() - if not issubclass(type(st), SFTPAttributes): - self._send_status(request_number, st, 'Unable to stat file') - return - length = st.st_size - start - if block_size == 0: - block_size = length - if block_size < 256: - self._send_status(request_number, SFTP_FAILURE, 'Block size too small') - return - - sum_out = '' - offset = start - while offset < start + length: - blocklen = min(block_size, start + length - offset) - # don't try to read more than about 64KB at a time - chunklen = min(blocklen, 65536) - count = 0 - hash_obj = alg.new() - while count < blocklen: - data = f.read(offset, chunklen) - if not type(data) is str: - self._send_status(request_number, data, 'Unable to hash file') - return - hash_obj.update(data) - count += len(data) - offset += count - sum_out += hash_obj.digest() - - msg = Message() - msg.add_int(request_number) - msg.add_string('check-file') - msg.add_string(algname) - msg.add_bytes(sum_out) - self._send_packet(CMD_EXTENDED_REPLY, str(msg)) - - def _convert_pflags(self, pflags): - "convert SFTP-style open() flags to python's os.open() flags" - if (pflags & SFTP_FLAG_READ) and (pflags & SFTP_FLAG_WRITE): - flags = os.O_RDWR - elif pflags & SFTP_FLAG_WRITE: - flags = os.O_WRONLY - else: - flags = os.O_RDONLY - if pflags & SFTP_FLAG_APPEND: - flags |= os.O_APPEND - if pflags & SFTP_FLAG_CREATE: - flags |= os.O_CREAT - if pflags & SFTP_FLAG_TRUNC: - flags |= os.O_TRUNC - if pflags & SFTP_FLAG_EXCL: - flags |= os.O_EXCL - return flags - - def _process(self, t, request_number, msg): - self._log(DEBUG, 'Request: %s' % CMD_NAMES[t]) - if t == CMD_OPEN: - path = msg.get_string() - flags = self._convert_pflags(msg.get_int()) - attr = SFTPAttributes._from_msg(msg) - self._send_handle_response(request_number, self.server.open(path, flags, attr)) - elif t == CMD_CLOSE: - handle = msg.get_string() - if handle in self.folder_table: - del self.folder_table[handle] - self._send_status(request_number, SFTP_OK) - return - if handle in self.file_table: - self.file_table[handle].close() - del self.file_table[handle] - self._send_status(request_number, SFTP_OK) - return - self._send_status(request_number, SFTP_BAD_MESSAGE, 'Invalid handle') - elif t == CMD_READ: - handle = msg.get_string() - offset = msg.get_int64() - length = msg.get_int() - if handle not in self.file_table: - self._send_status(request_number, SFTP_BAD_MESSAGE, 'Invalid handle') - return - data = self.file_table[handle].read(offset, length) - if type(data) is str: - if len(data) == 0: - self._send_status(request_number, SFTP_EOF) - else: - self._response(request_number, CMD_DATA, data) - else: - self._send_status(request_number, data) - elif t == CMD_WRITE: - handle = msg.get_string() - offset = msg.get_int64() - data = msg.get_string() - if handle not in self.file_table: - self._send_status(request_number, SFTP_BAD_MESSAGE, 'Invalid handle') - return - self._send_status(request_number, self.file_table[handle].write(offset, data)) - elif t == CMD_REMOVE: - path = msg.get_string() - self._send_status(request_number, self.server.remove(path)) - elif t == CMD_RENAME: - oldpath = msg.get_string() - newpath = msg.get_string() - self._send_status(request_number, self.server.rename(oldpath, newpath)) - elif t == CMD_MKDIR: - path = msg.get_string() - attr = SFTPAttributes._from_msg(msg) - self._send_status(request_number, self.server.mkdir(path, attr)) - elif t == CMD_RMDIR: - path = msg.get_string() - self._send_status(request_number, self.server.rmdir(path)) - elif t == CMD_OPENDIR: - path = msg.get_string() - self._open_folder(request_number, path) - return - elif t == CMD_READDIR: - handle = msg.get_string() - if handle not in self.folder_table: - self._send_status(request_number, SFTP_BAD_MESSAGE, 'Invalid handle') - return - folder = self.folder_table[handle] - self._read_folder(request_number, folder) - elif t == CMD_STAT: - path = msg.get_string() - resp = self.server.stat(path) - if issubclass(type(resp), SFTPAttributes): - self._response(request_number, CMD_ATTRS, resp) - else: - self._send_status(request_number, resp) - elif t == CMD_LSTAT: - path = msg.get_string() - resp = self.server.lstat(path) - if issubclass(type(resp), SFTPAttributes): - self._response(request_number, CMD_ATTRS, resp) - else: - self._send_status(request_number, resp) - elif t == CMD_FSTAT: - handle = msg.get_string() - if handle not in self.file_table: - self._send_status(request_number, SFTP_BAD_MESSAGE, 'Invalid handle') - return - resp = self.file_table[handle].stat() - if issubclass(type(resp), SFTPAttributes): - self._response(request_number, CMD_ATTRS, resp) - else: - self._send_status(request_number, resp) - elif t == CMD_SETSTAT: - path = msg.get_string() - attr = SFTPAttributes._from_msg(msg) - self._send_status(request_number, self.server.chattr(path, attr)) - elif t == CMD_FSETSTAT: - handle = msg.get_string() - attr = SFTPAttributes._from_msg(msg) - if handle not in self.file_table: - self._response(request_number, SFTP_BAD_MESSAGE, 'Invalid handle') - return - self._send_status(request_number, self.file_table[handle].chattr(attr)) - elif t == CMD_READLINK: - path = msg.get_string() - resp = self.server.readlink(path) - if type(resp) is str: - self._response(request_number, CMD_NAME, 1, resp, '', SFTPAttributes()) - else: - self._send_status(request_number, resp) - elif t == CMD_SYMLINK: - # the sftp 2 draft is incorrect here! path always follows target_path - target_path = msg.get_string() - path = msg.get_string() - self._send_status(request_number, self.server.symlink(target_path, path)) - elif t == CMD_REALPATH: - path = msg.get_string() - rpath = self.server.canonicalize(path) - self._response(request_number, CMD_NAME, 1, rpath, '', SFTPAttributes()) - elif t == CMD_EXTENDED: - tag = msg.get_string() - if tag == 'check-file': - self._check_file(request_number, msg) - else: - self._send_status(request_number, SFTP_OP_UNSUPPORTED) - else: - self._send_status(request_number, SFTP_OP_UNSUPPORTED) - - -from paramiko.sftp_handle import SFTPHandle diff --git a/contrib/site-packages/paramiko/sftp_si.py b/contrib/site-packages/paramiko/sftp_si.py deleted file mode 100644 index b0ee3c42..00000000 --- a/contrib/site-packages/paramiko/sftp_si.py +++ /dev/null @@ -1,310 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -L{SFTPServerInterface} is an interface to override for SFTP server support. -""" - -import os - -from paramiko.common import * -from paramiko.sftp import * - - -class SFTPServerInterface (object): - """ - This class defines an interface for controlling the behavior of paramiko - when using the L{SFTPServer} subsystem to provide an SFTP server. - - Methods on this class are called from the SFTP session's thread, so you can - block as long as necessary without affecting other sessions (even other - SFTP sessions). However, raising an exception will usually cause the SFTP - session to abruptly end, so you will usually want to catch exceptions and - return an appropriate error code. - - All paths are in string form instead of unicode because not all SFTP - clients & servers obey the requirement that paths be encoded in UTF-8. - """ - - def __init__ (self, server, *largs, **kwargs): - """ - Create a new SFTPServerInterface object. This method does nothing by - default and is meant to be overridden by subclasses. - - @param server: the server object associated with this channel and - SFTP subsystem - @type server: L{ServerInterface} - """ - super(SFTPServerInterface, self).__init__(*largs, **kwargs) - - def session_started(self): - """ - The SFTP server session has just started. This method is meant to be - overridden to perform any necessary setup before handling callbacks - from SFTP operations. - """ - pass - - def session_ended(self): - """ - The SFTP server session has just ended, either cleanly or via an - exception. This method is meant to be overridden to perform any - necessary cleanup before this C{SFTPServerInterface} object is - destroyed. - """ - pass - - def open(self, path, flags, attr): - """ - Open a file on the server and create a handle for future operations - on that file. On success, a new object subclassed from L{SFTPHandle} - should be returned. This handle will be used for future operations - on the file (read, write, etc). On failure, an error code such as - L{SFTP_PERMISSION_DENIED} should be returned. - - C{flags} contains the requested mode for opening (read-only, - write-append, etc) as a bitset of flags from the C{os} module: - - C{os.O_RDONLY} - - C{os.O_WRONLY} - - C{os.O_RDWR} - - C{os.O_APPEND} - - C{os.O_CREAT} - - C{os.O_TRUNC} - - C{os.O_EXCL} - (One of C{os.O_RDONLY}, C{os.O_WRONLY}, or C{os.O_RDWR} will always - be set.) - - The C{attr} object contains requested attributes of the file if it - has to be created. Some or all attribute fields may be missing if - the client didn't specify them. - - @note: The SFTP protocol defines all files to be in "binary" mode. - There is no equivalent to python's "text" mode. - - @param path: the requested path (relative or absolute) of the file - to be opened. - @type path: str - @param flags: flags or'd together from the C{os} module indicating the - requested mode for opening the file. - @type flags: int - @param attr: requested attributes of the file if it is newly created. - @type attr: L{SFTPAttributes} - @return: a new L{SFTPHandle} I{or error code}. - @rtype L{SFTPHandle} - """ - return SFTP_OP_UNSUPPORTED - - def list_folder(self, path): - """ - Return a list of files within a given folder. The C{path} will use - posix notation (C{"/"} separates folder names) and may be an absolute - or relative path. - - The list of files is expected to be a list of L{SFTPAttributes} - objects, which are similar in structure to the objects returned by - C{os.stat}. In addition, each object should have its C{filename} - field filled in, since this is important to a directory listing and - not normally present in C{os.stat} results. The method - L{SFTPAttributes.from_stat} will usually do what you want. - - In case of an error, you should return one of the C{SFTP_*} error - codes, such as L{SFTP_PERMISSION_DENIED}. - - @param path: the requested path (relative or absolute) to be listed. - @type path: str - @return: a list of the files in the given folder, using - L{SFTPAttributes} objects. - @rtype: list of L{SFTPAttributes} I{or error code} - - @note: You should normalize the given C{path} first (see the - C{os.path} module) and check appropriate permissions before returning - the list of files. Be careful of malicious clients attempting to use - relative paths to escape restricted folders, if you're doing a direct - translation from the SFTP server path to your local filesystem. - """ - return SFTP_OP_UNSUPPORTED - - def stat(self, path): - """ - Return an L{SFTPAttributes} object for a path on the server, or an - error code. If your server supports symbolic links (also known as - "aliases"), you should follow them. (L{lstat} is the corresponding - call that doesn't follow symlinks/aliases.) - - @param path: the requested path (relative or absolute) to fetch - file statistics for. - @type path: str - @return: an attributes object for the given file, or an SFTP error - code (like L{SFTP_PERMISSION_DENIED}). - @rtype: L{SFTPAttributes} I{or error code} - """ - return SFTP_OP_UNSUPPORTED - - def lstat(self, path): - """ - Return an L{SFTPAttributes} object for a path on the server, or an - error code. If your server supports symbolic links (also known as - "aliases"), you should I{not} follow them -- instead, you should - return data on the symlink or alias itself. (L{stat} is the - corresponding call that follows symlinks/aliases.) - - @param path: the requested path (relative or absolute) to fetch - file statistics for. - @type path: str - @return: an attributes object for the given file, or an SFTP error - code (like L{SFTP_PERMISSION_DENIED}). - @rtype: L{SFTPAttributes} I{or error code} - """ - return SFTP_OP_UNSUPPORTED - - def remove(self, path): - """ - Delete a file, if possible. - - @param path: the requested path (relative or absolute) of the file - to delete. - @type path: str - @return: an SFTP error code like L{SFTP_OK}. - @rtype: int - """ - return SFTP_OP_UNSUPPORTED - - def rename(self, oldpath, newpath): - """ - Rename (or move) a file. The SFTP specification implies that this - method can be used to move an existing file into a different folder, - and since there's no other (easy) way to move files via SFTP, it's - probably a good idea to implement "move" in this method too, even for - files that cross disk partition boundaries, if at all possible. - - @note: You should return an error if a file with the same name as - C{newpath} already exists. (The rename operation should be - non-desctructive.) - - @param oldpath: the requested path (relative or absolute) of the - existing file. - @type oldpath: str - @param newpath: the requested new path of the file. - @type newpath: str - @return: an SFTP error code like L{SFTP_OK}. - @rtype: int - """ - return SFTP_OP_UNSUPPORTED - - def mkdir(self, path, attr): - """ - Create a new directory with the given attributes. The C{attr} - object may be considered a "hint" and ignored. - - The C{attr} object will contain only those fields provided by the - client in its request, so you should use C{hasattr} to check for - the presense of fields before using them. In some cases, the C{attr} - object may be completely empty. - - @param path: requested path (relative or absolute) of the new - folder. - @type path: str - @param attr: requested attributes of the new folder. - @type attr: L{SFTPAttributes} - @return: an SFTP error code like L{SFTP_OK}. - @rtype: int - """ - return SFTP_OP_UNSUPPORTED - - def rmdir(self, path): - """ - Remove a directory if it exists. The C{path} should refer to an - existing, empty folder -- otherwise this method should return an - error. - - @param path: requested path (relative or absolute) of the folder - to remove. - @type path: str - @return: an SFTP error code like L{SFTP_OK}. - @rtype: int - """ - return SFTP_OP_UNSUPPORTED - - def chattr(self, path, attr): - """ - Change the attributes of a file. The C{attr} object will contain - only those fields provided by the client in its request, so you - should check for the presence of fields before using them. - - @param path: requested path (relative or absolute) of the file to - change. - @type path: str - @param attr: requested attributes to change on the file. - @type attr: L{SFTPAttributes} - @return: an error code like L{SFTP_OK}. - @rtype: int - """ - return SFTP_OP_UNSUPPORTED - - def canonicalize(self, path): - """ - Return the canonical form of a path on the server. For example, - if the server's home folder is C{/home/foo}, the path - C{"../betty"} would be canonicalized to C{"/home/betty"}. Note - the obvious security issues: if you're serving files only from a - specific folder, you probably don't want this method to reveal path - names outside that folder. - - You may find the python methods in C{os.path} useful, especially - C{os.path.normpath} and C{os.path.realpath}. - - The default implementation returns C{os.path.normpath('/' + path)}. - """ - if os.path.isabs(path): - out = os.path.normpath(path) - else: - out = os.path.normpath('/' + path) - if sys.platform == 'win32': - # on windows, normalize backslashes to sftp/posix format - out = out.replace('\\', '/') - return out - - def readlink(self, path): - """ - Return the target of a symbolic link (or shortcut) on the server. - If the specified path doesn't refer to a symbolic link, an error - should be returned. - - @param path: path (relative or absolute) of the symbolic link. - @type path: str - @return: the target path of the symbolic link, or an error code like - L{SFTP_NO_SUCH_FILE}. - @rtype: str I{or error code} - """ - return SFTP_OP_UNSUPPORTED - - def symlink(self, target_path, path): - """ - Create a symbolic link on the server, as new pathname C{path}, - with C{target_path} as the target of the link. - - @param target_path: path (relative or absolute) of the target for - this new symbolic link. - @type target_path: str - @param path: path (relative or absolute) of the symbolic link to - create. - @type path: str - @return: an error code like C{SFTP_OK}. - @rtype: int - """ - return SFTP_OP_UNSUPPORTED diff --git a/contrib/site-packages/paramiko/ssh_exception.py b/contrib/site-packages/paramiko/ssh_exception.py deleted file mode 100644 index b502b563..00000000 --- a/contrib/site-packages/paramiko/ssh_exception.py +++ /dev/null @@ -1,132 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -Exceptions defined by paramiko. -""" - - -class SSHException (Exception): - """ - Exception raised by failures in SSH2 protocol negotiation or logic errors. - """ - pass - - -class AuthenticationException (SSHException): - """ - Exception raised when authentication failed for some reason. It may be - possible to retry with different credentials. (Other classes specify more - specific reasons.) - - @since: 1.6 - """ - pass - - -class PasswordRequiredException (AuthenticationException): - """ - Exception raised when a password is needed to unlock a private key file. - """ - pass - - -class BadAuthenticationType (AuthenticationException): - """ - Exception raised when an authentication type (like password) is used, but - the server isn't allowing that type. (It may only allow public-key, for - example.) - - @ivar allowed_types: list of allowed authentication types provided by the - server (possible values are: C{"none"}, C{"password"}, and - C{"publickey"}). - @type allowed_types: list - - @since: 1.1 - """ - allowed_types = [] - - def __init__(self, explanation, types): - AuthenticationException.__init__(self, explanation) - self.allowed_types = types - - def __str__(self): - return SSHException.__str__(self) + ' (allowed_types=%r)' % self.allowed_types - - -class PartialAuthentication (AuthenticationException): - """ - An internal exception thrown in the case of partial authentication. - """ - allowed_types = [] - - def __init__(self, types): - AuthenticationException.__init__(self, 'partial authentication') - self.allowed_types = types - - -class ChannelException (SSHException): - """ - Exception raised when an attempt to open a new L{Channel} fails. - - @ivar code: the error code returned by the server - @type code: int - - @since: 1.6 - """ - def __init__(self, code, text): - SSHException.__init__(self, text) - self.code = code - - -class BadHostKeyException (SSHException): - """ - The host key given by the SSH server did not match what we were expecting. - - @ivar hostname: the hostname of the SSH server - @type hostname: str - @ivar key: the host key presented by the server - @type key: L{PKey} - @ivar expected_key: the host key expected - @type expected_key: L{PKey} - - @since: 1.6 - """ - def __init__(self, hostname, got_key, expected_key): - SSHException.__init__(self, 'Host key for server %s does not match!' % hostname) - self.hostname = hostname - self.key = got_key - self.expected_key = expected_key - - -class ProxyCommandFailure (SSHException): - """ - The "ProxyCommand" found in the .ssh/config file returned an error. - - @ivar command: The command line that is generating this exception. - @type command: str - @ivar error: The error captured from the proxy command output. - @type error: str - """ - def __init__(self, command, error): - SSHException.__init__(self, - '"ProxyCommand (%s)" returned non-zero exit status: %s' % ( - command, error - ) - ) - self.error = error diff --git a/contrib/site-packages/paramiko/transport.py b/contrib/site-packages/paramiko/transport.py deleted file mode 100644 index 3155d3f8..00000000 --- a/contrib/site-packages/paramiko/transport.py +++ /dev/null @@ -1,2158 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -L{Transport} handles the core SSH2 protocol. -""" - -import os -import socket -import string -import struct -import sys -import threading -import time -import weakref - -import paramiko -from paramiko import util -from paramiko.auth_handler import AuthHandler -from paramiko.channel import Channel -from paramiko.common import * -from paramiko.compress import ZlibCompressor, ZlibDecompressor -from paramiko.dsskey import DSSKey -from paramiko.kex_gex import KexGex -from paramiko.kex_group1 import KexGroup1 -from paramiko.message import Message -from paramiko.packet import Packetizer, NeedRekeyException -from paramiko.primes import ModulusPack -from paramiko.rsakey import RSAKey -from paramiko.ecdsakey import ECDSAKey -from paramiko.server import ServerInterface -from paramiko.sftp_client import SFTPClient -from paramiko.ssh_exception import (SSHException, BadAuthenticationType, - ChannelException, ProxyCommandFailure) -from paramiko.util import retry_on_signal - -from Crypto import Random -from Crypto.Cipher import Blowfish, AES, DES3, ARC4 -from Crypto.Hash import SHA, MD5 -try: - from Crypto.Util import Counter -except ImportError: - from paramiko.util import Counter - - -# for thread cleanup -_active_threads = [] -def _join_lingering_threads(): - for thr in _active_threads: - thr.stop_thread() -import atexit -atexit.register(_join_lingering_threads) - - -class SecurityOptions (object): - """ - Simple object containing the security preferences of an ssh transport. - These are tuples of acceptable ciphers, digests, key types, and key - exchange algorithms, listed in order of preference. - - Changing the contents and/or order of these fields affects the underlying - L{Transport} (but only if you change them before starting the session). - If you try to add an algorithm that paramiko doesn't recognize, - C{ValueError} will be raised. If you try to assign something besides a - tuple to one of the fields, C{TypeError} will be raised. - """ - __slots__ = [ 'ciphers', 'digests', 'key_types', 'kex', 'compression', '_transport' ] - - def __init__(self, transport): - self._transport = transport - - def __repr__(self): - """ - Returns a string representation of this object, for debugging. - - @rtype: str - """ - return '' % repr(self._transport) - - def _get_ciphers(self): - return self._transport._preferred_ciphers - - def _get_digests(self): - return self._transport._preferred_macs - - def _get_key_types(self): - return self._transport._preferred_keys - - def _get_kex(self): - return self._transport._preferred_kex - - def _get_compression(self): - return self._transport._preferred_compression - - def _set(self, name, orig, x): - if type(x) is list: - x = tuple(x) - if type(x) is not tuple: - raise TypeError('expected tuple or list') - possible = getattr(self._transport, orig).keys() - forbidden = filter(lambda n: n not in possible, x) - if len(forbidden) > 0: - raise ValueError('unknown cipher') - setattr(self._transport, name, x) - - def _set_ciphers(self, x): - self._set('_preferred_ciphers', '_cipher_info', x) - - def _set_digests(self, x): - self._set('_preferred_macs', '_mac_info', x) - - def _set_key_types(self, x): - self._set('_preferred_keys', '_key_info', x) - - def _set_kex(self, x): - self._set('_preferred_kex', '_kex_info', x) - - def _set_compression(self, x): - self._set('_preferred_compression', '_compression_info', x) - - ciphers = property(_get_ciphers, _set_ciphers, None, - "Symmetric encryption ciphers") - digests = property(_get_digests, _set_digests, None, - "Digest (one-way hash) algorithms") - key_types = property(_get_key_types, _set_key_types, None, - "Public-key algorithms") - kex = property(_get_kex, _set_kex, None, "Key exchange algorithms") - compression = property(_get_compression, _set_compression, None, - "Compression algorithms") - - -class ChannelMap (object): - def __init__(self): - # (id -> Channel) - self._map = weakref.WeakValueDictionary() - self._lock = threading.Lock() - - def put(self, chanid, chan): - self._lock.acquire() - try: - self._map[chanid] = chan - finally: - self._lock.release() - - def get(self, chanid): - self._lock.acquire() - try: - return self._map.get(chanid, None) - finally: - self._lock.release() - - def delete(self, chanid): - self._lock.acquire() - try: - try: - del self._map[chanid] - except KeyError: - pass - finally: - self._lock.release() - - def values(self): - self._lock.acquire() - try: - return self._map.values() - finally: - self._lock.release() - - def __len__(self): - self._lock.acquire() - try: - return len(self._map) - finally: - self._lock.release() - - -class Transport (threading.Thread): - """ - An SSH Transport attaches to a stream (usually a socket), negotiates an - encrypted session, authenticates, and then creates stream tunnels, called - L{Channel}s, across the session. Multiple channels can be multiplexed - across a single session (and often are, in the case of port forwardings). - """ - - _PROTO_ID = '2.0' - _CLIENT_ID = 'paramiko_%s' % (paramiko.__version__) - - _preferred_ciphers = ( 'aes128-ctr', 'aes256-ctr', 'aes128-cbc', 'blowfish-cbc', 'aes256-cbc', '3des-cbc', - 'arcfour128', 'arcfour256' ) - _preferred_macs = ( 'hmac-sha1', 'hmac-md5', 'hmac-sha1-96', 'hmac-md5-96' ) - _preferred_keys = ( 'ssh-rsa', 'ssh-dss', 'ecdsa-sha2-nistp256' ) - _preferred_kex = ( 'diffie-hellman-group1-sha1', 'diffie-hellman-group-exchange-sha1' ) - _preferred_compression = ( 'none', ) - - _cipher_info = { - 'aes128-ctr': { 'class': AES, 'mode': AES.MODE_CTR, 'block-size': 16, 'key-size': 16 }, - 'aes256-ctr': { 'class': AES, 'mode': AES.MODE_CTR, 'block-size': 16, 'key-size': 32 }, - 'blowfish-cbc': { 'class': Blowfish, 'mode': Blowfish.MODE_CBC, 'block-size': 8, 'key-size': 16 }, - 'aes128-cbc': { 'class': AES, 'mode': AES.MODE_CBC, 'block-size': 16, 'key-size': 16 }, - 'aes256-cbc': { 'class': AES, 'mode': AES.MODE_CBC, 'block-size': 16, 'key-size': 32 }, - '3des-cbc': { 'class': DES3, 'mode': DES3.MODE_CBC, 'block-size': 8, 'key-size': 24 }, - 'arcfour128': { 'class': ARC4, 'mode': None, 'block-size': 8, 'key-size': 16 }, - 'arcfour256': { 'class': ARC4, 'mode': None, 'block-size': 8, 'key-size': 32 }, - } - - _mac_info = { - 'hmac-sha1': { 'class': SHA, 'size': 20 }, - 'hmac-sha1-96': { 'class': SHA, 'size': 12 }, - 'hmac-md5': { 'class': MD5, 'size': 16 }, - 'hmac-md5-96': { 'class': MD5, 'size': 12 }, - } - - _key_info = { - 'ssh-rsa': RSAKey, - 'ssh-dss': DSSKey, - 'ecdsa-sha2-nistp256': ECDSAKey, - } - - _kex_info = { - 'diffie-hellman-group1-sha1': KexGroup1, - 'diffie-hellman-group-exchange-sha1': KexGex, - } - - _compression_info = { - # zlib@openssh.com is just zlib, but only turned on after a successful - # authentication. openssh servers may only offer this type because - # they've had troubles with security holes in zlib in the past. - 'zlib@openssh.com': ( ZlibCompressor, ZlibDecompressor ), - 'zlib': ( ZlibCompressor, ZlibDecompressor ), - 'none': ( None, None ), - } - - - _modulus_pack = None - - def __init__(self, sock): - """ - Create a new SSH session over an existing socket, or socket-like - object. This only creates the Transport object; it doesn't begin the - SSH session yet. Use L{connect} or L{start_client} to begin a client - session, or L{start_server} to begin a server session. - - If the object is not actually a socket, it must have the following - methods: - - C{send(str)}: Writes from 1 to C{len(str)} bytes, and - returns an int representing the number of bytes written. Returns - 0 or raises C{EOFError} if the stream has been closed. - - C{recv(int)}: Reads from 1 to C{int} bytes and returns them as a - string. Returns 0 or raises C{EOFError} if the stream has been - closed. - - C{close()}: Closes the socket. - - C{settimeout(n)}: Sets a (float) timeout on I/O operations. - - For ease of use, you may also pass in an address (as a tuple) or a host - string as the C{sock} argument. (A host string is a hostname with an - optional port (separated by C{":"}) which will be converted into a - tuple of C{(hostname, port)}.) A socket will be connected to this - address and used for communication. Exceptions from the C{socket} call - may be thrown in this case. - - @param sock: a socket or socket-like object to create the session over. - @type sock: socket - """ - if isinstance(sock, (str, unicode)): - # convert "host:port" into (host, port) - hl = sock.split(':', 1) - if len(hl) == 1: - sock = (hl[0], 22) - else: - sock = (hl[0], int(hl[1])) - if type(sock) is tuple: - # connect to the given (host, port) - hostname, port = sock - reason = 'No suitable address family' - for (family, socktype, proto, canonname, sockaddr) in socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM): - if socktype == socket.SOCK_STREAM: - af = family - addr = sockaddr - sock = socket.socket(af, socket.SOCK_STREAM) - try: - retry_on_signal(lambda: sock.connect((hostname, port))) - except socket.error, e: - reason = str(e) - else: - break - else: - raise SSHException( - 'Unable to connect to %s: %s' % (hostname, reason)) - # okay, normal socket-ish flow here... - threading.Thread.__init__(self) - self.setDaemon(True) - self.rng = rng - self.sock = sock - # Python < 2.3 doesn't have the settimeout method - RogerB - try: - # we set the timeout so we can check self.active periodically to - # see if we should bail. socket.timeout exception is never - # propagated. - self.sock.settimeout(0.1) - except AttributeError: - pass - - # negotiated crypto parameters - self.packetizer = Packetizer(sock) - self.local_version = 'SSH-' + self._PROTO_ID + '-' + self._CLIENT_ID - self.remote_version = '' - self.local_cipher = self.remote_cipher = '' - self.local_kex_init = self.remote_kex_init = None - self.local_mac = self.remote_mac = None - self.local_compression = self.remote_compression = None - self.session_id = None - self.host_key_type = None - self.host_key = None - - # state used during negotiation - self.kex_engine = None - self.H = None - self.K = None - - self.active = False - self.initial_kex_done = False - self.in_kex = False - self.authenticated = False - self._expected_packet = tuple() - self.lock = threading.Lock() # synchronization (always higher level than write_lock) - - # tracking open channels - self._channels = ChannelMap() - self.channel_events = { } # (id -> Event) - self.channels_seen = { } # (id -> True) - self._channel_counter = 1 - self.window_size = 65536 - self.max_packet_size = 34816 - self._forward_agent_handler = None - self._x11_handler = None - self._tcp_handler = None - - self.saved_exception = None - self.clear_to_send = threading.Event() - self.clear_to_send_lock = threading.Lock() - self.clear_to_send_timeout = 30.0 - self.log_name = 'paramiko.transport' - self.logger = util.get_logger(self.log_name) - self.packetizer.set_log(self.logger) - self.auth_handler = None - self.global_response = None # response Message from an arbitrary global request - self.completion_event = None # user-defined event callbacks - self.banner_timeout = 15 # how long (seconds) to wait for the SSH banner - - # server mode: - self.server_mode = False - self.server_object = None - self.server_key_dict = { } - self.server_accepts = [ ] - self.server_accept_cv = threading.Condition(self.lock) - self.subsystem_table = { } - - def __repr__(self): - """ - Returns a string representation of this object, for debugging. - - @rtype: str - """ - out = '} or - L{auth_publickey }. - - @note: L{connect} is a simpler method for connecting as a client. - - @note: After calling this method (or L{start_server} or L{connect}), - you should no longer directly read from or write to the original - socket object. - - @param event: an event to trigger when negotiation is complete - (optional) - @type event: threading.Event - - @raise SSHException: if negotiation fails (and no C{event} was passed - in) - """ - self.active = True - if event is not None: - # async, return immediately and let the app poll for completion - self.completion_event = event - self.start() - return - - # synchronous, wait for a result - self.completion_event = event = threading.Event() - self.start() - Random.atfork() - while True: - event.wait(0.1) - if not self.active: - e = self.get_exception() - if e is not None: - raise e - raise SSHException('Negotiation failed.') - if event.isSet(): - break - - def start_server(self, event=None, server=None): - """ - Negotiate a new SSH2 session as a server. This is the first step after - creating a new L{Transport} and setting up your server host key(s). A - separate thread is created for protocol negotiation. - - If an event is passed in, this method returns immediately. When - negotiation is done (successful or not), the given C{Event} will - be triggered. On failure, L{is_active} will return C{False}. - - (Since 1.4) If C{event} is C{None}, this method will not return until - negotation is done. On success, the method returns normally. - Otherwise an SSHException is raised. - - After a successful negotiation, the client will need to authenticate. - Override the methods - L{get_allowed_auths }, - L{check_auth_none }, - L{check_auth_password }, and - L{check_auth_publickey } in the - given C{server} object to control the authentication process. - - After a successful authentication, the client should request to open - a channel. Override - L{check_channel_request } in the - given C{server} object to allow channels to be opened. - - @note: After calling this method (or L{start_client} or L{connect}), - you should no longer directly read from or write to the original - socket object. - - @param event: an event to trigger when negotiation is complete. - @type event: threading.Event - @param server: an object used to perform authentication and create - L{Channel}s. - @type server: L{server.ServerInterface} - - @raise SSHException: if negotiation fails (and no C{event} was passed - in) - """ - if server is None: - server = ServerInterface() - self.server_mode = True - self.server_object = server - self.active = True - if event is not None: - # async, return immediately and let the app poll for completion - self.completion_event = event - self.start() - return - - # synchronous, wait for a result - self.completion_event = event = threading.Event() - self.start() - while True: - event.wait(0.1) - if not self.active: - e = self.get_exception() - if e is not None: - raise e - raise SSHException('Negotiation failed.') - if event.isSet(): - break - - def add_server_key(self, key): - """ - Add a host key to the list of keys used for server mode. When behaving - as a server, the host key is used to sign certain packets during the - SSH2 negotiation, so that the client can trust that we are who we say - we are. Because this is used for signing, the key must contain private - key info, not just the public half. Only one key of each type (RSA or - DSS) is kept. - - @param key: the host key to add, usually an L{RSAKey } or - L{DSSKey }. - @type key: L{PKey } - """ - self.server_key_dict[key.get_name()] = key - - def get_server_key(self): - """ - Return the active host key, in server mode. After negotiating with the - client, this method will return the negotiated host key. If only one - type of host key was set with L{add_server_key}, that's the only key - that will ever be returned. But in cases where you have set more than - one type of host key (for example, an RSA key and a DSS key), the key - type will be negotiated by the client, and this method will return the - key of the type agreed on. If the host key has not been negotiated - yet, C{None} is returned. In client mode, the behavior is undefined. - - @return: host key of the type negotiated by the client, or C{None}. - @rtype: L{PKey } - """ - try: - return self.server_key_dict[self.host_key_type] - except KeyError: - pass - return None - - def load_server_moduli(filename=None): - """ - I{(optional)} - Load a file of prime moduli for use in doing group-exchange key - negotiation in server mode. It's a rather obscure option and can be - safely ignored. - - In server mode, the remote client may request "group-exchange" key - negotiation, which asks the server to send a random prime number that - fits certain criteria. These primes are pretty difficult to compute, - so they can't be generated on demand. But many systems contain a file - of suitable primes (usually named something like C{/etc/ssh/moduli}). - If you call C{load_server_moduli} and it returns C{True}, then this - file of primes has been loaded and we will support "group-exchange" in - server mode. Otherwise server mode will just claim that it doesn't - support that method of key negotiation. - - @param filename: optional path to the moduli file, if you happen to - know that it's not in a standard location. - @type filename: str - @return: True if a moduli file was successfully loaded; False - otherwise. - @rtype: bool - - @note: This has no effect when used in client mode. - """ - Transport._modulus_pack = ModulusPack(rng) - # places to look for the openssh "moduli" file - file_list = [ '/etc/ssh/moduli', '/usr/local/etc/moduli' ] - if filename is not None: - file_list.insert(0, filename) - for fn in file_list: - try: - Transport._modulus_pack.read_file(fn) - return True - except IOError: - pass - # none succeeded - Transport._modulus_pack = None - return False - load_server_moduli = staticmethod(load_server_moduli) - - def close(self): - """ - Close this session, and any open channels that are tied to it. - """ - if not self.active: - return - self.stop_thread() - for chan in self._channels.values(): - chan._unlink() - self.sock.close() - - def get_remote_server_key(self): - """ - Return the host key of the server (in client mode). - - @note: Previously this call returned a tuple of (key type, key string). - You can get the same effect by calling - L{PKey.get_name } for the key type, and - C{str(key)} for the key string. - - @raise SSHException: if no session is currently active. - - @return: public key of the remote server - @rtype: L{PKey } - """ - if (not self.active) or (not self.initial_kex_done): - raise SSHException('No existing session') - return self.host_key - - def is_active(self): - """ - Return true if this session is active (open). - - @return: True if the session is still active (open); False if the - session is closed - @rtype: bool - """ - return self.active - - def open_session(self): - """ - Request a new channel to the server, of type C{"session"}. This - is just an alias for C{open_channel('session')}. - - @return: a new L{Channel} - @rtype: L{Channel} - - @raise SSHException: if the request is rejected or the session ends - prematurely - """ - return self.open_channel('session') - - def open_x11_channel(self, src_addr=None): - """ - Request a new channel to the client, of type C{"x11"}. This - is just an alias for C{open_channel('x11', src_addr=src_addr)}. - - @param src_addr: the source address of the x11 server (port is the - x11 port, ie. 6010) - @type src_addr: (str, int) - @return: a new L{Channel} - @rtype: L{Channel} - - @raise SSHException: if the request is rejected or the session ends - prematurely - """ - return self.open_channel('x11', src_addr=src_addr) - - def open_forward_agent_channel(self): - """ - Request a new channel to the client, of type - C{"auth-agent@openssh.com"}. - - This is just an alias for C{open_channel('auth-agent@openssh.com')}. - @return: a new L{Channel} - @rtype: L{Channel} - - @raise SSHException: if the request is rejected or the session ends - prematurely - """ - return self.open_channel('auth-agent@openssh.com') - - def open_forwarded_tcpip_channel(self, (src_addr, src_port), (dest_addr, dest_port)): - """ - Request a new channel back to the client, of type C{"forwarded-tcpip"}. - This is used after a client has requested port forwarding, for sending - incoming connections back to the client. - - @param src_addr: originator's address - @param src_port: originator's port - @param dest_addr: local (server) connected address - @param dest_port: local (server) connected port - """ - return self.open_channel('forwarded-tcpip', (dest_addr, dest_port), (src_addr, src_port)) - - def open_channel(self, kind, dest_addr=None, src_addr=None): - """ - Request a new channel to the server. L{Channel}s are socket-like - objects used for the actual transfer of data across the session. - You may only request a channel after negotiating encryption (using - L{connect} or L{start_client}) and authenticating. - - @param kind: the kind of channel requested (usually C{"session"}, - C{"forwarded-tcpip"}, C{"direct-tcpip"}, or C{"x11"}) - @type kind: str - @param dest_addr: the destination address of this port forwarding, - if C{kind} is C{"forwarded-tcpip"} or C{"direct-tcpip"} (ignored - for other channel types) - @type dest_addr: (str, int) - @param src_addr: the source address of this port forwarding, if - C{kind} is C{"forwarded-tcpip"}, C{"direct-tcpip"}, or C{"x11"} - @type src_addr: (str, int) - @return: a new L{Channel} on success - @rtype: L{Channel} - - @raise SSHException: if the request is rejected or the session ends - prematurely - """ - if not self.active: - raise SSHException('SSH session not active') - self.lock.acquire() - try: - chanid = self._next_channel() - m = Message() - m.add_byte(chr(MSG_CHANNEL_OPEN)) - m.add_string(kind) - m.add_int(chanid) - m.add_int(self.window_size) - m.add_int(self.max_packet_size) - if (kind == 'forwarded-tcpip') or (kind == 'direct-tcpip'): - m.add_string(dest_addr[0]) - m.add_int(dest_addr[1]) - m.add_string(src_addr[0]) - m.add_int(src_addr[1]) - elif kind == 'x11': - m.add_string(src_addr[0]) - m.add_int(src_addr[1]) - chan = Channel(chanid) - self._channels.put(chanid, chan) - self.channel_events[chanid] = event = threading.Event() - self.channels_seen[chanid] = True - chan._set_transport(self) - chan._set_window(self.window_size, self.max_packet_size) - finally: - self.lock.release() - self._send_user_message(m) - while True: - event.wait(0.1); - if not self.active: - e = self.get_exception() - if e is None: - e = SSHException('Unable to open channel.') - raise e - if event.isSet(): - break - chan = self._channels.get(chanid) - if chan is not None: - return chan - e = self.get_exception() - if e is None: - e = SSHException('Unable to open channel.') - raise e - - def request_port_forward(self, address, port, handler=None): - """ - Ask the server to forward TCP connections from a listening port on - the server, across this SSH session. - - If a handler is given, that handler is called from a different thread - whenever a forwarded connection arrives. The handler parameters are:: - - handler(channel, (origin_addr, origin_port), (server_addr, server_port)) - - where C{server_addr} and C{server_port} are the address and port that - the server was listening on. - - If no handler is set, the default behavior is to send new incoming - forwarded connections into the accept queue, to be picked up via - L{accept}. - - @param address: the address to bind when forwarding - @type address: str - @param port: the port to forward, or 0 to ask the server to allocate - any port - @type port: int - @param handler: optional handler for incoming forwarded connections - @type handler: function(Channel, (str, int), (str, int)) - @return: the port # allocated by the server - @rtype: int - - @raise SSHException: if the server refused the TCP forward request - """ - if not self.active: - raise SSHException('SSH session not active') - address = str(address) - port = int(port) - response = self.global_request('tcpip-forward', (address, port), wait=True) - if response is None: - raise SSHException('TCP forwarding request denied') - if port == 0: - port = response.get_int() - if handler is None: - def default_handler(channel, (src_addr, src_port), (dest_addr, dest_port)): - self._queue_incoming_channel(channel) - handler = default_handler - self._tcp_handler = handler - return port - - def cancel_port_forward(self, address, port): - """ - Ask the server to cancel a previous port-forwarding request. No more - connections to the given address & port will be forwarded across this - ssh connection. - - @param address: the address to stop forwarding - @type address: str - @param port: the port to stop forwarding - @type port: int - """ - if not self.active: - return - self._tcp_handler = None - self.global_request('cancel-tcpip-forward', (address, port), wait=True) - - def open_sftp_client(self): - """ - Create an SFTP client channel from an open transport. On success, - an SFTP session will be opened with the remote host, and a new - SFTPClient object will be returned. - - @return: a new L{SFTPClient} object, referring to an sftp session - (channel) across this transport - @rtype: L{SFTPClient} - """ - return SFTPClient.from_transport(self) - - def send_ignore(self, bytes=None): - """ - Send a junk packet across the encrypted link. This is sometimes used - to add "noise" to a connection to confuse would-be attackers. It can - also be used as a keep-alive for long lived connections traversing - firewalls. - - @param bytes: the number of random bytes to send in the payload of the - ignored packet -- defaults to a random number from 10 to 41. - @type bytes: int - """ - m = Message() - m.add_byte(chr(MSG_IGNORE)) - if bytes is None: - bytes = (ord(rng.read(1)) % 32) + 10 - m.add_bytes(rng.read(bytes)) - self._send_user_message(m) - - def renegotiate_keys(self): - """ - Force this session to switch to new keys. Normally this is done - automatically after the session hits a certain number of packets or - bytes sent or received, but this method gives you the option of forcing - new keys whenever you want. Negotiating new keys causes a pause in - traffic both ways as the two sides swap keys and do computations. This - method returns when the session has switched to new keys. - - @raise SSHException: if the key renegotiation failed (which causes the - session to end) - """ - self.completion_event = threading.Event() - self._send_kex_init() - while True: - self.completion_event.wait(0.1) - if not self.active: - e = self.get_exception() - if e is not None: - raise e - raise SSHException('Negotiation failed.') - if self.completion_event.isSet(): - break - return - - def set_keepalive(self, interval): - """ - Turn on/off keepalive packets (default is off). If this is set, after - C{interval} seconds without sending any data over the connection, a - "keepalive" packet will be sent (and ignored by the remote host). This - can be useful to keep connections alive over a NAT, for example. - - @param interval: seconds to wait before sending a keepalive packet (or - 0 to disable keepalives). - @type interval: int - """ - self.packetizer.set_keepalive(interval, - lambda x=weakref.proxy(self): x.global_request('keepalive@lag.net', wait=False)) - - def global_request(self, kind, data=None, wait=True): - """ - Make a global request to the remote host. These are normally - extensions to the SSH2 protocol. - - @param kind: name of the request. - @type kind: str - @param data: an optional tuple containing additional data to attach - to the request. - @type data: tuple - @param wait: C{True} if this method should not return until a response - is received; C{False} otherwise. - @type wait: bool - @return: a L{Message} containing possible additional data if the - request was successful (or an empty L{Message} if C{wait} was - C{False}); C{None} if the request was denied. - @rtype: L{Message} - """ - if wait: - self.completion_event = threading.Event() - m = Message() - m.add_byte(chr(MSG_GLOBAL_REQUEST)) - m.add_string(kind) - m.add_boolean(wait) - if data is not None: - m.add(*data) - self._log(DEBUG, 'Sending global request "%s"' % kind) - self._send_user_message(m) - if not wait: - return None - while True: - self.completion_event.wait(0.1) - if not self.active: - return None - if self.completion_event.isSet(): - break - return self.global_response - - def accept(self, timeout=None): - """ - Return the next channel opened by the client over this transport, in - server mode. If no channel is opened before the given timeout, C{None} - is returned. - - @param timeout: seconds to wait for a channel, or C{None} to wait - forever - @type timeout: int - @return: a new Channel opened by the client - @rtype: L{Channel} - """ - self.lock.acquire() - try: - if len(self.server_accepts) > 0: - chan = self.server_accepts.pop(0) - else: - self.server_accept_cv.wait(timeout) - if len(self.server_accepts) > 0: - chan = self.server_accepts.pop(0) - else: - # timeout - chan = None - finally: - self.lock.release() - return chan - - def connect(self, hostkey=None, username='', password=None, pkey=None): - """ - Negotiate an SSH2 session, and optionally verify the server's host key - and authenticate using a password or private key. This is a shortcut - for L{start_client}, L{get_remote_server_key}, and - L{Transport.auth_password} or L{Transport.auth_publickey}. Use those - methods if you want more control. - - You can use this method immediately after creating a Transport to - negotiate encryption with a server. If it fails, an exception will be - thrown. On success, the method will return cleanly, and an encrypted - session exists. You may immediately call L{open_channel} or - L{open_session} to get a L{Channel} object, which is used for data - transfer. - - @note: If you fail to supply a password or private key, this method may - succeed, but a subsequent L{open_channel} or L{open_session} call may - fail because you haven't authenticated yet. - - @param hostkey: the host key expected from the server, or C{None} if - you don't want to do host key verification. - @type hostkey: L{PKey} - @param username: the username to authenticate as. - @type username: str - @param password: a password to use for authentication, if you want to - use password authentication; otherwise C{None}. - @type password: str - @param pkey: a private key to use for authentication, if you want to - use private key authentication; otherwise C{None}. - @type pkey: L{PKey} - - @raise SSHException: if the SSH2 negotiation fails, the host key - supplied by the server is incorrect, or authentication fails. - """ - if hostkey is not None: - self._preferred_keys = [ hostkey.get_name() ] - - self.start_client() - - # check host key if we were given one - if (hostkey is not None): - key = self.get_remote_server_key() - if (key.get_name() != hostkey.get_name()) or (str(key) != str(hostkey)): - self._log(DEBUG, 'Bad host key from server') - self._log(DEBUG, 'Expected: %s: %s' % (hostkey.get_name(), repr(str(hostkey)))) - self._log(DEBUG, 'Got : %s: %s' % (key.get_name(), repr(str(key)))) - raise SSHException('Bad host key from server') - self._log(DEBUG, 'Host key verified (%s)' % hostkey.get_name()) - - if (pkey is not None) or (password is not None): - if password is not None: - self._log(DEBUG, 'Attempting password auth...') - self.auth_password(username, password) - else: - self._log(DEBUG, 'Attempting public-key auth...') - self.auth_publickey(username, pkey) - - return - - def get_exception(self): - """ - Return any exception that happened during the last server request. - This can be used to fetch more specific error information after using - calls like L{start_client}. The exception (if any) is cleared after - this call. - - @return: an exception, or C{None} if there is no stored exception. - @rtype: Exception - - @since: 1.1 - """ - self.lock.acquire() - try: - e = self.saved_exception - self.saved_exception = None - return e - finally: - self.lock.release() - - def set_subsystem_handler(self, name, handler, *larg, **kwarg): - """ - Set the handler class for a subsystem in server mode. If a request - for this subsystem is made on an open ssh channel later, this handler - will be constructed and called -- see L{SubsystemHandler} for more - detailed documentation. - - Any extra parameters (including keyword arguments) are saved and - passed to the L{SubsystemHandler} constructor later. - - @param name: name of the subsystem. - @type name: str - @param handler: subclass of L{SubsystemHandler} that handles this - subsystem. - @type handler: class - """ - try: - self.lock.acquire() - self.subsystem_table[name] = (handler, larg, kwarg) - finally: - self.lock.release() - - def is_authenticated(self): - """ - Return true if this session is active and authenticated. - - @return: True if the session is still open and has been authenticated - successfully; False if authentication failed and/or the session is - closed. - @rtype: bool - """ - return self.active and (self.auth_handler is not None) and self.auth_handler.is_authenticated() - - def get_username(self): - """ - Return the username this connection is authenticated for. If the - session is not authenticated (or authentication failed), this method - returns C{None}. - - @return: username that was authenticated, or C{None}. - @rtype: string - """ - if not self.active or (self.auth_handler is None): - return None - return self.auth_handler.get_username() - - def auth_none(self, username): - """ - Try to authenticate to the server using no authentication at all. - This will almost always fail. It may be useful for determining the - list of authentication types supported by the server, by catching the - L{BadAuthenticationType} exception raised. - - @param username: the username to authenticate as - @type username: string - @return: list of auth types permissible for the next stage of - authentication (normally empty) - @rtype: list - - @raise BadAuthenticationType: if "none" authentication isn't allowed - by the server for this user - @raise SSHException: if the authentication failed due to a network - error - - @since: 1.5 - """ - if (not self.active) or (not self.initial_kex_done): - raise SSHException('No existing session') - my_event = threading.Event() - self.auth_handler = AuthHandler(self) - self.auth_handler.auth_none(username, my_event) - return self.auth_handler.wait_for_response(my_event) - - def auth_password(self, username, password, event=None, fallback=True): - """ - Authenticate to the server using a password. The username and password - are sent over an encrypted link. - - If an C{event} is passed in, this method will return immediately, and - the event will be triggered once authentication succeeds or fails. On - success, L{is_authenticated} will return C{True}. On failure, you may - use L{get_exception} to get more detailed error information. - - Since 1.1, if no event is passed, this method will block until the - authentication succeeds or fails. On failure, an exception is raised. - Otherwise, the method simply returns. - - Since 1.5, if no event is passed and C{fallback} is C{True} (the - default), if the server doesn't support plain password authentication - but does support so-called "keyboard-interactive" mode, an attempt - will be made to authenticate using this interactive mode. If it fails, - the normal exception will be thrown as if the attempt had never been - made. This is useful for some recent Gentoo and Debian distributions, - which turn off plain password authentication in a misguided belief - that interactive authentication is "more secure". (It's not.) - - If the server requires multi-step authentication (which is very rare), - this method will return a list of auth types permissible for the next - step. Otherwise, in the normal case, an empty list is returned. - - @param username: the username to authenticate as - @type username: str - @param password: the password to authenticate with - @type password: str or unicode - @param event: an event to trigger when the authentication attempt is - complete (whether it was successful or not) - @type event: threading.Event - @param fallback: C{True} if an attempt at an automated "interactive" - password auth should be made if the server doesn't support normal - password auth - @type fallback: bool - @return: list of auth types permissible for the next stage of - authentication (normally empty) - @rtype: list - - @raise BadAuthenticationType: if password authentication isn't - allowed by the server for this user (and no event was passed in) - @raise AuthenticationException: if the authentication failed (and no - event was passed in) - @raise SSHException: if there was a network error - """ - if (not self.active) or (not self.initial_kex_done): - # we should never try to send the password unless we're on a secure link - raise SSHException('No existing session') - if event is None: - my_event = threading.Event() - else: - my_event = event - self.auth_handler = AuthHandler(self) - self.auth_handler.auth_password(username, password, my_event) - if event is not None: - # caller wants to wait for event themselves - return [] - try: - return self.auth_handler.wait_for_response(my_event) - except BadAuthenticationType, x: - # if password auth isn't allowed, but keyboard-interactive *is*, try to fudge it - if not fallback or ('keyboard-interactive' not in x.allowed_types): - raise - try: - def handler(title, instructions, fields): - if len(fields) > 1: - raise SSHException('Fallback authentication failed.') - if len(fields) == 0: - # for some reason, at least on os x, a 2nd request will - # be made with zero fields requested. maybe it's just - # to try to fake out automated scripting of the exact - # type we're doing here. *shrug* :) - return [] - return [ password ] - return self.auth_interactive(username, handler) - except SSHException, ignored: - # attempt failed; just raise the original exception - raise x - return None - - def auth_publickey(self, username, key, event=None): - """ - Authenticate to the server using a private key. The key is used to - sign data from the server, so it must include the private part. - - If an C{event} is passed in, this method will return immediately, and - the event will be triggered once authentication succeeds or fails. On - success, L{is_authenticated} will return C{True}. On failure, you may - use L{get_exception} to get more detailed error information. - - Since 1.1, if no event is passed, this method will block until the - authentication succeeds or fails. On failure, an exception is raised. - Otherwise, the method simply returns. - - If the server requires multi-step authentication (which is very rare), - this method will return a list of auth types permissible for the next - step. Otherwise, in the normal case, an empty list is returned. - - @param username: the username to authenticate as - @type username: string - @param key: the private key to authenticate with - @type key: L{PKey } - @param event: an event to trigger when the authentication attempt is - complete (whether it was successful or not) - @type event: threading.Event - @return: list of auth types permissible for the next stage of - authentication (normally empty) - @rtype: list - - @raise BadAuthenticationType: if public-key authentication isn't - allowed by the server for this user (and no event was passed in) - @raise AuthenticationException: if the authentication failed (and no - event was passed in) - @raise SSHException: if there was a network error - """ - if (not self.active) or (not self.initial_kex_done): - # we should never try to authenticate unless we're on a secure link - raise SSHException('No existing session') - if event is None: - my_event = threading.Event() - else: - my_event = event - self.auth_handler = AuthHandler(self) - self.auth_handler.auth_publickey(username, key, my_event) - if event is not None: - # caller wants to wait for event themselves - return [] - return self.auth_handler.wait_for_response(my_event) - - def auth_interactive(self, username, handler, submethods=''): - """ - Authenticate to the server interactively. A handler is used to answer - arbitrary questions from the server. On many servers, this is just a - dumb wrapper around PAM. - - This method will block until the authentication succeeds or fails, - peroidically calling the handler asynchronously to get answers to - authentication questions. The handler may be called more than once - if the server continues to ask questions. - - The handler is expected to be a callable that will handle calls of the - form: C{handler(title, instructions, prompt_list)}. The C{title} is - meant to be a dialog-window title, and the C{instructions} are user - instructions (both are strings). C{prompt_list} will be a list of - prompts, each prompt being a tuple of C{(str, bool)}. The string is - the prompt and the boolean indicates whether the user text should be - echoed. - - A sample call would thus be: - C{handler('title', 'instructions', [('Password:', False)])}. - - The handler should return a list or tuple of answers to the server's - questions. - - If the server requires multi-step authentication (which is very rare), - this method will return a list of auth types permissible for the next - step. Otherwise, in the normal case, an empty list is returned. - - @param username: the username to authenticate as - @type username: string - @param handler: a handler for responding to server questions - @type handler: callable - @param submethods: a string list of desired submethods (optional) - @type submethods: str - @return: list of auth types permissible for the next stage of - authentication (normally empty). - @rtype: list - - @raise BadAuthenticationType: if public-key authentication isn't - allowed by the server for this user - @raise AuthenticationException: if the authentication failed - @raise SSHException: if there was a network error - - @since: 1.5 - """ - if (not self.active) or (not self.initial_kex_done): - # we should never try to authenticate unless we're on a secure link - raise SSHException('No existing session') - my_event = threading.Event() - self.auth_handler = AuthHandler(self) - self.auth_handler.auth_interactive(username, handler, my_event, submethods) - return self.auth_handler.wait_for_response(my_event) - - def set_log_channel(self, name): - """ - Set the channel for this transport's logging. The default is - C{"paramiko.transport"} but it can be set to anything you want. - (See the C{logging} module for more info.) SSH Channels will log - to a sub-channel of the one specified. - - @param name: new channel name for logging - @type name: str - - @since: 1.1 - """ - self.log_name = name - self.logger = util.get_logger(name) - self.packetizer.set_log(self.logger) - - def get_log_channel(self): - """ - Return the channel name used for this transport's logging. - - @return: channel name. - @rtype: str - - @since: 1.2 - """ - return self.log_name - - def set_hexdump(self, hexdump): - """ - Turn on/off logging a hex dump of protocol traffic at DEBUG level in - the logs. Normally you would want this off (which is the default), - but if you are debugging something, it may be useful. - - @param hexdump: C{True} to log protocol traffix (in hex) to the log; - C{False} otherwise. - @type hexdump: bool - """ - self.packetizer.set_hexdump(hexdump) - - def get_hexdump(self): - """ - Return C{True} if the transport is currently logging hex dumps of - protocol traffic. - - @return: C{True} if hex dumps are being logged - @rtype: bool - - @since: 1.4 - """ - return self.packetizer.get_hexdump() - - def use_compression(self, compress=True): - """ - Turn on/off compression. This will only have an affect before starting - the transport (ie before calling L{connect}, etc). By default, - compression is off since it negatively affects interactive sessions. - - @param compress: C{True} to ask the remote client/server to compress - traffic; C{False} to refuse compression - @type compress: bool - - @since: 1.5.2 - """ - if compress: - self._preferred_compression = ( 'zlib@openssh.com', 'zlib', 'none' ) - else: - self._preferred_compression = ( 'none', ) - - def getpeername(self): - """ - Return the address of the remote side of this Transport, if possible. - This is effectively a wrapper around C{'getpeername'} on the underlying - socket. If the socket-like object has no C{'getpeername'} method, - then C{("unknown", 0)} is returned. - - @return: the address if the remote host, if known - @rtype: tuple(str, int) - """ - gp = getattr(self.sock, 'getpeername', None) - if gp is None: - return ('unknown', 0) - return gp() - - def stop_thread(self): - self.active = False - self.packetizer.close() - while self.isAlive(): - self.join(10) - - - ### internals... - - - def _log(self, level, msg, *args): - if issubclass(type(msg), list): - for m in msg: - self.logger.log(level, m) - else: - self.logger.log(level, msg, *args) - - def _get_modulus_pack(self): - "used by KexGex to find primes for group exchange" - return self._modulus_pack - - def _next_channel(self): - "you are holding the lock" - chanid = self._channel_counter - while self._channels.get(chanid) is not None: - self._channel_counter = (self._channel_counter + 1) & 0xffffff - chanid = self._channel_counter - self._channel_counter = (self._channel_counter + 1) & 0xffffff - return chanid - - def _unlink_channel(self, chanid): - "used by a Channel to remove itself from the active channel list" - self._channels.delete(chanid) - - def _send_message(self, data): - self.packetizer.send_message(data) - - def _send_user_message(self, data): - """ - send a message, but block if we're in key negotiation. this is used - for user-initiated requests. - """ - start = time.time() - while True: - self.clear_to_send.wait(0.1) - if not self.active: - self._log(DEBUG, 'Dropping user packet because connection is dead.') - return - self.clear_to_send_lock.acquire() - if self.clear_to_send.isSet(): - break - self.clear_to_send_lock.release() - if time.time() > start + self.clear_to_send_timeout: - raise SSHException('Key-exchange timed out waiting for key negotiation') - try: - self._send_message(data) - finally: - self.clear_to_send_lock.release() - - def _set_K_H(self, k, h): - "used by a kex object to set the K (root key) and H (exchange hash)" - self.K = k - self.H = h - if self.session_id == None: - self.session_id = h - - def _expect_packet(self, *ptypes): - "used by a kex object to register the next packet type it expects to see" - self._expected_packet = tuple(ptypes) - - def _verify_key(self, host_key, sig): - key = self._key_info[self.host_key_type](Message(host_key)) - if key is None: - raise SSHException('Unknown host key type') - if not key.verify_ssh_sig(self.H, Message(sig)): - raise SSHException('Signature verification (%s) failed.' % self.host_key_type) - self.host_key = key - - def _compute_key(self, id, nbytes): - "id is 'A' - 'F' for the various keys used by ssh" - m = Message() - m.add_mpint(self.K) - m.add_bytes(self.H) - m.add_byte(id) - m.add_bytes(self.session_id) - out = sofar = SHA.new(str(m)).digest() - while len(out) < nbytes: - m = Message() - m.add_mpint(self.K) - m.add_bytes(self.H) - m.add_bytes(sofar) - digest = SHA.new(str(m)).digest() - out += digest - sofar += digest - return out[:nbytes] - - def _get_cipher(self, name, key, iv): - if name not in self._cipher_info: - raise SSHException('Unknown client cipher ' + name) - if name in ('arcfour128', 'arcfour256'): - # arcfour cipher - cipher = self._cipher_info[name]['class'].new(key) - # as per RFC 4345, the first 1536 bytes of keystream - # generated by the cipher MUST be discarded - cipher.encrypt(" " * 1536) - return cipher - elif name.endswith("-ctr"): - # CTR modes, we need a counter - counter = Counter.new(nbits=self._cipher_info[name]['block-size'] * 8, initial_value=util.inflate_long(iv, True)) - return self._cipher_info[name]['class'].new(key, self._cipher_info[name]['mode'], iv, counter) - else: - return self._cipher_info[name]['class'].new(key, self._cipher_info[name]['mode'], iv) - - def _set_forward_agent_handler(self, handler): - if handler is None: - def default_handler(channel): - self._queue_incoming_channel(channel) - self._forward_agent_handler = default_handler - else: - self._forward_agent_handler = handler - - def _set_x11_handler(self, handler): - # only called if a channel has turned on x11 forwarding - if handler is None: - # by default, use the same mechanism as accept() - def default_handler(channel, (src_addr, src_port)): - self._queue_incoming_channel(channel) - self._x11_handler = default_handler - else: - self._x11_handler = handler - - def _queue_incoming_channel(self, channel): - self.lock.acquire() - try: - self.server_accepts.append(channel) - self.server_accept_cv.notify() - finally: - self.lock.release() - - def run(self): - # (use the exposed "run" method, because if we specify a thread target - # of a private method, threading.Thread will keep a reference to it - # indefinitely, creating a GC cycle and not letting Transport ever be - # GC'd. it's a bug in Thread.) - - # Hold reference to 'sys' so we can test sys.modules to detect - # interpreter shutdown. - self.sys = sys - - # Required to prevent RNG errors when running inside many subprocess - # containers. - Random.atfork() - - # Hold reference to 'sys' so we can test sys.modules to detect - # interpreter shutdown. - self.sys = sys - - # active=True occurs before the thread is launched, to avoid a race - _active_threads.append(self) - if self.server_mode: - self._log(DEBUG, 'starting thread (server mode): %s' % hex(long(id(self)) & 0xffffffffL)) - else: - self._log(DEBUG, 'starting thread (client mode): %s' % hex(long(id(self)) & 0xffffffffL)) - try: - try: - self.packetizer.write_all(self.local_version + '\r\n') - self._check_banner() - self._send_kex_init() - self._expect_packet(MSG_KEXINIT) - - while self.active: - if self.packetizer.need_rekey() and not self.in_kex: - self._send_kex_init() - try: - ptype, m = self.packetizer.read_message() - except NeedRekeyException: - continue - if ptype == MSG_IGNORE: - continue - elif ptype == MSG_DISCONNECT: - self._parse_disconnect(m) - self.active = False - self.packetizer.close() - break - elif ptype == MSG_DEBUG: - self._parse_debug(m) - continue - if len(self._expected_packet) > 0: - if ptype not in self._expected_packet: - raise SSHException('Expecting packet from %r, got %d' % (self._expected_packet, ptype)) - self._expected_packet = tuple() - if (ptype >= 30) and (ptype <= 39): - self.kex_engine.parse_next(ptype, m) - continue - - if ptype in self._handler_table: - self._handler_table[ptype](self, m) - elif ptype in self._channel_handler_table: - chanid = m.get_int() - chan = self._channels.get(chanid) - if chan is not None: - self._channel_handler_table[ptype](chan, m) - elif chanid in self.channels_seen: - self._log(DEBUG, 'Ignoring message for dead channel %d' % chanid) - else: - self._log(ERROR, 'Channel request for unknown channel %d' % chanid) - self.active = False - self.packetizer.close() - elif (self.auth_handler is not None) and (ptype in self.auth_handler._handler_table): - self.auth_handler._handler_table[ptype](self.auth_handler, m) - else: - self._log(WARNING, 'Oops, unhandled type %d' % ptype) - msg = Message() - msg.add_byte(chr(MSG_UNIMPLEMENTED)) - msg.add_int(m.seqno) - self._send_message(msg) - except SSHException, e: - self._log(ERROR, 'Exception: ' + str(e)) - self._log(ERROR, util.tb_strings()) - self.saved_exception = e - except EOFError, e: - self._log(DEBUG, 'EOF in transport thread') - #self._log(DEBUG, util.tb_strings()) - self.saved_exception = e - except socket.error, e: - if type(e.args) is tuple: - emsg = '%s (%d)' % (e.args[1], e.args[0]) - else: - emsg = e.args - self._log(ERROR, 'Socket exception: ' + emsg) - self.saved_exception = e - except Exception, e: - self._log(ERROR, 'Unknown exception: ' + str(e)) - self._log(ERROR, util.tb_strings()) - self.saved_exception = e - _active_threads.remove(self) - for chan in self._channels.values(): - chan._unlink() - if self.active: - self.active = False - self.packetizer.close() - if self.completion_event != None: - self.completion_event.set() - if self.auth_handler is not None: - self.auth_handler.abort() - for event in self.channel_events.values(): - event.set() - try: - self.lock.acquire() - self.server_accept_cv.notify() - finally: - self.lock.release() - self.sock.close() - except: - # Don't raise spurious 'NoneType has no attribute X' errors when we - # wake up during interpreter shutdown. Or rather -- raise - # everything *if* sys.modules (used as a convenient sentinel) - # appears to still exist. - if self.sys.modules is not None: - raise - - - ### protocol stages - - - def _negotiate_keys(self, m): - # throws SSHException on anything unusual - self.clear_to_send_lock.acquire() - try: - self.clear_to_send.clear() - finally: - self.clear_to_send_lock.release() - if self.local_kex_init == None: - # remote side wants to renegotiate - self._send_kex_init() - self._parse_kex_init(m) - self.kex_engine.start_kex() - - def _check_banner(self): - # this is slow, but we only have to do it once - for i in range(100): - # give them 15 seconds for the first line, then just 2 seconds - # each additional line. (some sites have very high latency.) - if i == 0: - timeout = self.banner_timeout - else: - timeout = 2 - try: - buf = self.packetizer.readline(timeout) - except ProxyCommandFailure: - raise - except Exception, x: - raise SSHException('Error reading SSH protocol banner' + str(x)) - if buf[:4] == 'SSH-': - break - self._log(DEBUG, 'Banner: ' + buf) - if buf[:4] != 'SSH-': - raise SSHException('Indecipherable protocol version "' + buf + '"') - # save this server version string for later - self.remote_version = buf - # pull off any attached comment - comment = '' - i = string.find(buf, ' ') - if i >= 0: - comment = buf[i+1:] - buf = buf[:i] - # parse out version string and make sure it matches - segs = buf.split('-', 2) - if len(segs) < 3: - raise SSHException('Invalid SSH banner') - version = segs[1] - client = segs[2] - if version != '1.99' and version != '2.0': - raise SSHException('Incompatible version (%s instead of 2.0)' % (version,)) - self._log(INFO, 'Connected (version %s, client %s)' % (version, client)) - - def _send_kex_init(self): - """ - announce to the other side that we'd like to negotiate keys, and what - kind of key negotiation we support. - """ - self.clear_to_send_lock.acquire() - try: - self.clear_to_send.clear() - finally: - self.clear_to_send_lock.release() - self.in_kex = True - if self.server_mode: - if (self._modulus_pack is None) and ('diffie-hellman-group-exchange-sha1' in self._preferred_kex): - # can't do group-exchange if we don't have a pack of potential primes - pkex = list(self.get_security_options().kex) - pkex.remove('diffie-hellman-group-exchange-sha1') - self.get_security_options().kex = pkex - available_server_keys = filter(self.server_key_dict.keys().__contains__, - self._preferred_keys) - else: - available_server_keys = self._preferred_keys - - m = Message() - m.add_byte(chr(MSG_KEXINIT)) - m.add_bytes(rng.read(16)) - m.add_list(self._preferred_kex) - m.add_list(available_server_keys) - m.add_list(self._preferred_ciphers) - m.add_list(self._preferred_ciphers) - m.add_list(self._preferred_macs) - m.add_list(self._preferred_macs) - m.add_list(self._preferred_compression) - m.add_list(self._preferred_compression) - m.add_string('') - m.add_string('') - m.add_boolean(False) - m.add_int(0) - # save a copy for later (needed to compute a hash) - self.local_kex_init = str(m) - self._send_message(m) - - def _parse_kex_init(self, m): - cookie = m.get_bytes(16) - kex_algo_list = m.get_list() - server_key_algo_list = m.get_list() - client_encrypt_algo_list = m.get_list() - server_encrypt_algo_list = m.get_list() - client_mac_algo_list = m.get_list() - server_mac_algo_list = m.get_list() - client_compress_algo_list = m.get_list() - server_compress_algo_list = m.get_list() - client_lang_list = m.get_list() - server_lang_list = m.get_list() - kex_follows = m.get_boolean() - unused = m.get_int() - - self._log(DEBUG, 'kex algos:' + str(kex_algo_list) + ' server key:' + str(server_key_algo_list) + \ - ' client encrypt:' + str(client_encrypt_algo_list) + \ - ' server encrypt:' + str(server_encrypt_algo_list) + \ - ' client mac:' + str(client_mac_algo_list) + \ - ' server mac:' + str(server_mac_algo_list) + \ - ' client compress:' + str(client_compress_algo_list) + \ - ' server compress:' + str(server_compress_algo_list) + \ - ' client lang:' + str(client_lang_list) + \ - ' server lang:' + str(server_lang_list) + \ - ' kex follows?' + str(kex_follows)) - - # as a server, we pick the first item in the client's list that we support. - # as a client, we pick the first item in our list that the server supports. - if self.server_mode: - agreed_kex = filter(self._preferred_kex.__contains__, kex_algo_list) - else: - agreed_kex = filter(kex_algo_list.__contains__, self._preferred_kex) - if len(agreed_kex) == 0: - raise SSHException('Incompatible ssh peer (no acceptable kex algorithm)') - self.kex_engine = self._kex_info[agreed_kex[0]](self) - - if self.server_mode: - available_server_keys = filter(self.server_key_dict.keys().__contains__, - self._preferred_keys) - agreed_keys = filter(available_server_keys.__contains__, server_key_algo_list) - else: - agreed_keys = filter(server_key_algo_list.__contains__, self._preferred_keys) - if len(agreed_keys) == 0: - raise SSHException('Incompatible ssh peer (no acceptable host key)') - self.host_key_type = agreed_keys[0] - if self.server_mode and (self.get_server_key() is None): - raise SSHException('Incompatible ssh peer (can\'t match requested host key type)') - - if self.server_mode: - agreed_local_ciphers = filter(self._preferred_ciphers.__contains__, - server_encrypt_algo_list) - agreed_remote_ciphers = filter(self._preferred_ciphers.__contains__, - client_encrypt_algo_list) - else: - agreed_local_ciphers = filter(client_encrypt_algo_list.__contains__, - self._preferred_ciphers) - agreed_remote_ciphers = filter(server_encrypt_algo_list.__contains__, - self._preferred_ciphers) - if (len(agreed_local_ciphers) == 0) or (len(agreed_remote_ciphers) == 0): - raise SSHException('Incompatible ssh server (no acceptable ciphers)') - self.local_cipher = agreed_local_ciphers[0] - self.remote_cipher = agreed_remote_ciphers[0] - self._log(DEBUG, 'Ciphers agreed: local=%s, remote=%s' % (self.local_cipher, self.remote_cipher)) - - if self.server_mode: - agreed_remote_macs = filter(self._preferred_macs.__contains__, client_mac_algo_list) - agreed_local_macs = filter(self._preferred_macs.__contains__, server_mac_algo_list) - else: - agreed_local_macs = filter(client_mac_algo_list.__contains__, self._preferred_macs) - agreed_remote_macs = filter(server_mac_algo_list.__contains__, self._preferred_macs) - if (len(agreed_local_macs) == 0) or (len(agreed_remote_macs) == 0): - raise SSHException('Incompatible ssh server (no acceptable macs)') - self.local_mac = agreed_local_macs[0] - self.remote_mac = agreed_remote_macs[0] - - if self.server_mode: - agreed_remote_compression = filter(self._preferred_compression.__contains__, client_compress_algo_list) - agreed_local_compression = filter(self._preferred_compression.__contains__, server_compress_algo_list) - else: - agreed_local_compression = filter(client_compress_algo_list.__contains__, self._preferred_compression) - agreed_remote_compression = filter(server_compress_algo_list.__contains__, self._preferred_compression) - if (len(agreed_local_compression) == 0) or (len(agreed_remote_compression) == 0): - raise SSHException('Incompatible ssh server (no acceptable compression) %r %r %r' % (agreed_local_compression, agreed_remote_compression, self._preferred_compression)) - self.local_compression = agreed_local_compression[0] - self.remote_compression = agreed_remote_compression[0] - - self._log(DEBUG, 'using kex %s; server key type %s; cipher: local %s, remote %s; mac: local %s, remote %s; compression: local %s, remote %s' % - (agreed_kex[0], self.host_key_type, self.local_cipher, self.remote_cipher, self.local_mac, - self.remote_mac, self.local_compression, self.remote_compression)) - - # save for computing hash later... - # now wait! openssh has a bug (and others might too) where there are - # actually some extra bytes (one NUL byte in openssh's case) added to - # the end of the packet but not parsed. turns out we need to throw - # away those bytes because they aren't part of the hash. - self.remote_kex_init = chr(MSG_KEXINIT) + m.get_so_far() - - def _activate_inbound(self): - "switch on newly negotiated encryption parameters for inbound traffic" - block_size = self._cipher_info[self.remote_cipher]['block-size'] - if self.server_mode: - IV_in = self._compute_key('A', block_size) - key_in = self._compute_key('C', self._cipher_info[self.remote_cipher]['key-size']) - else: - IV_in = self._compute_key('B', block_size) - key_in = self._compute_key('D', self._cipher_info[self.remote_cipher]['key-size']) - engine = self._get_cipher(self.remote_cipher, key_in, IV_in) - mac_size = self._mac_info[self.remote_mac]['size'] - mac_engine = self._mac_info[self.remote_mac]['class'] - # initial mac keys are done in the hash's natural size (not the potentially truncated - # transmission size) - if self.server_mode: - mac_key = self._compute_key('E', mac_engine.digest_size) - else: - mac_key = self._compute_key('F', mac_engine.digest_size) - self.packetizer.set_inbound_cipher(engine, block_size, mac_engine, mac_size, mac_key) - compress_in = self._compression_info[self.remote_compression][1] - if (compress_in is not None) and ((self.remote_compression != 'zlib@openssh.com') or self.authenticated): - self._log(DEBUG, 'Switching on inbound compression ...') - self.packetizer.set_inbound_compressor(compress_in()) - - def _activate_outbound(self): - "switch on newly negotiated encryption parameters for outbound traffic" - m = Message() - m.add_byte(chr(MSG_NEWKEYS)) - self._send_message(m) - block_size = self._cipher_info[self.local_cipher]['block-size'] - if self.server_mode: - IV_out = self._compute_key('B', block_size) - key_out = self._compute_key('D', self._cipher_info[self.local_cipher]['key-size']) - else: - IV_out = self._compute_key('A', block_size) - key_out = self._compute_key('C', self._cipher_info[self.local_cipher]['key-size']) - engine = self._get_cipher(self.local_cipher, key_out, IV_out) - mac_size = self._mac_info[self.local_mac]['size'] - mac_engine = self._mac_info[self.local_mac]['class'] - # initial mac keys are done in the hash's natural size (not the potentially truncated - # transmission size) - if self.server_mode: - mac_key = self._compute_key('F', mac_engine.digest_size) - else: - mac_key = self._compute_key('E', mac_engine.digest_size) - sdctr = self.local_cipher.endswith('-ctr') - self.packetizer.set_outbound_cipher(engine, block_size, mac_engine, mac_size, mac_key, sdctr) - compress_out = self._compression_info[self.local_compression][0] - if (compress_out is not None) and ((self.local_compression != 'zlib@openssh.com') or self.authenticated): - self._log(DEBUG, 'Switching on outbound compression ...') - self.packetizer.set_outbound_compressor(compress_out()) - if not self.packetizer.need_rekey(): - self.in_kex = False - # we always expect to receive NEWKEYS now - self._expect_packet(MSG_NEWKEYS) - - def _auth_trigger(self): - self.authenticated = True - # delayed initiation of compression - if self.local_compression == 'zlib@openssh.com': - compress_out = self._compression_info[self.local_compression][0] - self._log(DEBUG, 'Switching on outbound compression ...') - self.packetizer.set_outbound_compressor(compress_out()) - if self.remote_compression == 'zlib@openssh.com': - compress_in = self._compression_info[self.remote_compression][1] - self._log(DEBUG, 'Switching on inbound compression ...') - self.packetizer.set_inbound_compressor(compress_in()) - - def _parse_newkeys(self, m): - self._log(DEBUG, 'Switch to new keys ...') - self._activate_inbound() - # can also free a bunch of stuff here - self.local_kex_init = self.remote_kex_init = None - self.K = None - self.kex_engine = None - if self.server_mode and (self.auth_handler is None): - # create auth handler for server mode - self.auth_handler = AuthHandler(self) - if not self.initial_kex_done: - # this was the first key exchange - self.initial_kex_done = True - # send an event? - if self.completion_event != None: - self.completion_event.set() - # it's now okay to send data again (if this was a re-key) - if not self.packetizer.need_rekey(): - self.in_kex = False - self.clear_to_send_lock.acquire() - try: - self.clear_to_send.set() - finally: - self.clear_to_send_lock.release() - return - - def _parse_disconnect(self, m): - code = m.get_int() - desc = m.get_string() - self._log(INFO, 'Disconnect (code %d): %s' % (code, desc)) - - def _parse_global_request(self, m): - kind = m.get_string() - self._log(DEBUG, 'Received global request "%s"' % kind) - want_reply = m.get_boolean() - if not self.server_mode: - self._log(DEBUG, 'Rejecting "%s" global request from server.' % kind) - ok = False - elif kind == 'tcpip-forward': - address = m.get_string() - port = m.get_int() - ok = self.server_object.check_port_forward_request(address, port) - if ok != False: - ok = (ok,) - elif kind == 'cancel-tcpip-forward': - address = m.get_string() - port = m.get_int() - self.server_object.cancel_port_forward_request(address, port) - ok = True - else: - ok = self.server_object.check_global_request(kind, m) - extra = () - if type(ok) is tuple: - extra = ok - ok = True - if want_reply: - msg = Message() - if ok: - msg.add_byte(chr(MSG_REQUEST_SUCCESS)) - msg.add(*extra) - else: - msg.add_byte(chr(MSG_REQUEST_FAILURE)) - self._send_message(msg) - - def _parse_request_success(self, m): - self._log(DEBUG, 'Global request successful.') - self.global_response = m - if self.completion_event is not None: - self.completion_event.set() - - def _parse_request_failure(self, m): - self._log(DEBUG, 'Global request denied.') - self.global_response = None - if self.completion_event is not None: - self.completion_event.set() - - def _parse_channel_open_success(self, m): - chanid = m.get_int() - server_chanid = m.get_int() - server_window_size = m.get_int() - server_max_packet_size = m.get_int() - chan = self._channels.get(chanid) - if chan is None: - self._log(WARNING, 'Success for unrequested channel! [??]') - return - self.lock.acquire() - try: - chan._set_remote_channel(server_chanid, server_window_size, server_max_packet_size) - self._log(INFO, 'Secsh channel %d opened.' % chanid) - if chanid in self.channel_events: - self.channel_events[chanid].set() - del self.channel_events[chanid] - finally: - self.lock.release() - return - - def _parse_channel_open_failure(self, m): - chanid = m.get_int() - reason = m.get_int() - reason_str = m.get_string() - lang = m.get_string() - reason_text = CONNECTION_FAILED_CODE.get(reason, '(unknown code)') - self._log(INFO, 'Secsh channel %d open FAILED: %s: %s' % (chanid, reason_str, reason_text)) - self.lock.acquire() - try: - self.saved_exception = ChannelException(reason, reason_text) - if chanid in self.channel_events: - self._channels.delete(chanid) - if chanid in self.channel_events: - self.channel_events[chanid].set() - del self.channel_events[chanid] - finally: - self.lock.release() - return - - def _parse_channel_open(self, m): - kind = m.get_string() - chanid = m.get_int() - initial_window_size = m.get_int() - max_packet_size = m.get_int() - reject = False - if (kind == 'auth-agent@openssh.com') and (self._forward_agent_handler is not None): - self._log(DEBUG, 'Incoming forward agent connection') - self.lock.acquire() - try: - my_chanid = self._next_channel() - finally: - self.lock.release() - elif (kind == 'x11') and (self._x11_handler is not None): - origin_addr = m.get_string() - origin_port = m.get_int() - self._log(DEBUG, 'Incoming x11 connection from %s:%d' % (origin_addr, origin_port)) - self.lock.acquire() - try: - my_chanid = self._next_channel() - finally: - self.lock.release() - elif (kind == 'forwarded-tcpip') and (self._tcp_handler is not None): - server_addr = m.get_string() - server_port = m.get_int() - origin_addr = m.get_string() - origin_port = m.get_int() - self._log(DEBUG, 'Incoming tcp forwarded connection from %s:%d' % (origin_addr, origin_port)) - self.lock.acquire() - try: - my_chanid = self._next_channel() - finally: - self.lock.release() - elif not self.server_mode: - self._log(DEBUG, 'Rejecting "%s" channel request from server.' % kind) - reject = True - reason = OPEN_FAILED_ADMINISTRATIVELY_PROHIBITED - else: - self.lock.acquire() - try: - my_chanid = self._next_channel() - finally: - self.lock.release() - if kind == 'direct-tcpip': - # handle direct-tcpip requests comming from the client - dest_addr = m.get_string() - dest_port = m.get_int() - origin_addr = m.get_string() - origin_port = m.get_int() - reason = self.server_object.check_channel_direct_tcpip_request( - my_chanid, (origin_addr, origin_port), - (dest_addr, dest_port)) - else: - reason = self.server_object.check_channel_request(kind, my_chanid) - if reason != OPEN_SUCCEEDED: - self._log(DEBUG, 'Rejecting "%s" channel request from client.' % kind) - reject = True - if reject: - msg = Message() - msg.add_byte(chr(MSG_CHANNEL_OPEN_FAILURE)) - msg.add_int(chanid) - msg.add_int(reason) - msg.add_string('') - msg.add_string('en') - self._send_message(msg) - return - - chan = Channel(my_chanid) - self.lock.acquire() - try: - self._channels.put(my_chanid, chan) - self.channels_seen[my_chanid] = True - chan._set_transport(self) - chan._set_window(self.window_size, self.max_packet_size) - chan._set_remote_channel(chanid, initial_window_size, max_packet_size) - finally: - self.lock.release() - m = Message() - m.add_byte(chr(MSG_CHANNEL_OPEN_SUCCESS)) - m.add_int(chanid) - m.add_int(my_chanid) - m.add_int(self.window_size) - m.add_int(self.max_packet_size) - self._send_message(m) - self._log(INFO, 'Secsh channel %d (%s) opened.', my_chanid, kind) - if kind == 'auth-agent@openssh.com': - self._forward_agent_handler(chan) - elif kind == 'x11': - self._x11_handler(chan, (origin_addr, origin_port)) - elif kind == 'forwarded-tcpip': - chan.origin_addr = (origin_addr, origin_port) - self._tcp_handler(chan, (origin_addr, origin_port), (server_addr, server_port)) - else: - self._queue_incoming_channel(chan) - - def _parse_debug(self, m): - always_display = m.get_boolean() - msg = m.get_string() - lang = m.get_string() - self._log(DEBUG, 'Debug msg: ' + util.safe_string(msg)) - - def _get_subsystem_handler(self, name): - try: - self.lock.acquire() - if name not in self.subsystem_table: - return (None, [], {}) - return self.subsystem_table[name] - finally: - self.lock.release() - - _handler_table = { - MSG_NEWKEYS: _parse_newkeys, - MSG_GLOBAL_REQUEST: _parse_global_request, - MSG_REQUEST_SUCCESS: _parse_request_success, - MSG_REQUEST_FAILURE: _parse_request_failure, - MSG_CHANNEL_OPEN_SUCCESS: _parse_channel_open_success, - MSG_CHANNEL_OPEN_FAILURE: _parse_channel_open_failure, - MSG_CHANNEL_OPEN: _parse_channel_open, - MSG_KEXINIT: _negotiate_keys, - } - - _channel_handler_table = { - MSG_CHANNEL_SUCCESS: Channel._request_success, - MSG_CHANNEL_FAILURE: Channel._request_failed, - MSG_CHANNEL_DATA: Channel._feed, - MSG_CHANNEL_EXTENDED_DATA: Channel._feed_extended, - MSG_CHANNEL_WINDOW_ADJUST: Channel._window_adjust, - MSG_CHANNEL_REQUEST: Channel._handle_request, - MSG_CHANNEL_EOF: Channel._handle_eof, - MSG_CHANNEL_CLOSE: Channel._handle_close, - } diff --git a/contrib/site-packages/paramiko/util.py b/contrib/site-packages/paramiko/util.py deleted file mode 100644 index 85ee6b06..00000000 --- a/contrib/site-packages/paramiko/util.py +++ /dev/null @@ -1,311 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -Useful functions used by the rest of paramiko. -""" - -from __future__ import generators - -import array -from binascii import hexlify, unhexlify -import errno -import sys -import struct -import traceback -import threading - -from paramiko.common import * -from paramiko.config import SSHConfig - - -# Change by RogerB - python < 2.3 doesn't have enumerate so we implement it -if sys.version_info < (2,3): - class enumerate: - def __init__ (self, sequence): - self.sequence = sequence - def __iter__ (self): - count = 0 - for item in self.sequence: - yield (count, item) - count += 1 - - -def inflate_long(s, always_positive=False): - "turns a normalized byte string into a long-int (adapted from Crypto.Util.number)" - out = 0L - negative = 0 - if not always_positive and (len(s) > 0) and (ord(s[0]) >= 0x80): - negative = 1 - if len(s) % 4: - filler = '\x00' - if negative: - filler = '\xff' - s = filler * (4 - len(s) % 4) + s - for i in range(0, len(s), 4): - out = (out << 32) + struct.unpack('>I', s[i:i+4])[0] - if negative: - out -= (1L << (8 * len(s))) - return out - -def deflate_long(n, add_sign_padding=True): - "turns a long-int into a normalized byte string (adapted from Crypto.Util.number)" - # after much testing, this algorithm was deemed to be the fastest - s = '' - n = long(n) - while (n != 0) and (n != -1): - s = struct.pack('>I', n & 0xffffffffL) + s - n = n >> 32 - # strip off leading zeros, FFs - for i in enumerate(s): - if (n == 0) and (i[1] != '\000'): - break - if (n == -1) and (i[1] != '\xff'): - break - else: - # degenerate case, n was either 0 or -1 - i = (0,) - if n == 0: - s = '\000' - else: - s = '\xff' - s = s[i[0]:] - if add_sign_padding: - if (n == 0) and (ord(s[0]) >= 0x80): - s = '\x00' + s - if (n == -1) and (ord(s[0]) < 0x80): - s = '\xff' + s - return s - -def format_binary_weird(data): - out = '' - for i in enumerate(data): - out += '%02X' % ord(i[1]) - if i[0] % 2: - out += ' ' - if i[0] % 16 == 15: - out += '\n' - return out - -def format_binary(data, prefix=''): - x = 0 - out = [] - while len(data) > x + 16: - out.append(format_binary_line(data[x:x+16])) - x += 16 - if x < len(data): - out.append(format_binary_line(data[x:])) - return [prefix + x for x in out] - -def format_binary_line(data): - left = ' '.join(['%02X' % ord(c) for c in data]) - right = ''.join([('.%c..' % c)[(ord(c)+63)//95] for c in data]) - return '%-50s %s' % (left, right) - -def hexify(s): - return hexlify(s).upper() - -def unhexify(s): - return unhexlify(s) - -def safe_string(s): - out = '' - for c in s: - if (ord(c) >= 32) and (ord(c) <= 127): - out += c - else: - out += '%%%02X' % ord(c) - return out - -# ''.join([['%%%02X' % ord(c), c][(ord(c) >= 32) and (ord(c) <= 127)] for c in s]) - -def bit_length(n): - norm = deflate_long(n, 0) - hbyte = ord(norm[0]) - if hbyte == 0: - return 1 - bitlen = len(norm) * 8 - while not (hbyte & 0x80): - hbyte <<= 1 - bitlen -= 1 - return bitlen - -def tb_strings(): - return ''.join(traceback.format_exception(*sys.exc_info())).split('\n') - -def generate_key_bytes(hashclass, salt, key, nbytes): - """ - Given a password, passphrase, or other human-source key, scramble it - through a secure hash into some keyworthy bytes. This specific algorithm - is used for encrypting/decrypting private key files. - - @param hashclass: class from L{Crypto.Hash} that can be used as a secure - hashing function (like C{MD5} or C{SHA}). - @type hashclass: L{Crypto.Hash} - @param salt: data to salt the hash with. - @type salt: string - @param key: human-entered password or passphrase. - @type key: string - @param nbytes: number of bytes to generate. - @type nbytes: int - @return: key data - @rtype: string - """ - keydata = '' - digest = '' - if len(salt) > 8: - salt = salt[:8] - while nbytes > 0: - hash_obj = hashclass.new() - if len(digest) > 0: - hash_obj.update(digest) - hash_obj.update(key) - hash_obj.update(salt) - digest = hash_obj.digest() - size = min(nbytes, len(digest)) - keydata += digest[:size] - nbytes -= size - return keydata - -def load_host_keys(filename): - """ - Read a file of known SSH host keys, in the format used by openssh, and - return a compound dict of C{hostname -> keytype ->} L{PKey }. - The hostname may be an IP address or DNS name. The keytype will be either - C{"ssh-rsa"} or C{"ssh-dss"}. - - This type of file unfortunately doesn't exist on Windows, but on posix, - it will usually be stored in C{os.path.expanduser("~/.ssh/known_hosts")}. - - Since 1.5.3, this is just a wrapper around L{HostKeys}. - - @param filename: name of the file to read host keys from - @type filename: str - @return: dict of host keys, indexed by hostname and then keytype - @rtype: dict(hostname, dict(keytype, L{PKey })) - """ - from paramiko.hostkeys import HostKeys - return HostKeys(filename) - -def parse_ssh_config(file_obj): - """ - Provided only as a backward-compatible wrapper around L{SSHConfig}. - """ - config = SSHConfig() - config.parse(file_obj) - return config - -def lookup_ssh_host_config(hostname, config): - """ - Provided only as a backward-compatible wrapper around L{SSHConfig}. - """ - return config.lookup(hostname) - -def mod_inverse(x, m): - # it's crazy how small python can make this function. - u1, u2, u3 = 1, 0, m - v1, v2, v3 = 0, 1, x - - while v3 > 0: - q = u3 // v3 - u1, v1 = v1, u1 - v1 * q - u2, v2 = v2, u2 - v2 * q - u3, v3 = v3, u3 - v3 * q - if u2 < 0: - u2 += m - return u2 - -_g_thread_ids = {} -_g_thread_counter = 0 -_g_thread_lock = threading.Lock() -def get_thread_id(): - global _g_thread_ids, _g_thread_counter, _g_thread_lock - tid = id(threading.currentThread()) - try: - return _g_thread_ids[tid] - except KeyError: - _g_thread_lock.acquire() - try: - _g_thread_counter += 1 - ret = _g_thread_ids[tid] = _g_thread_counter - finally: - _g_thread_lock.release() - return ret - -def log_to_file(filename, level=DEBUG): - "send paramiko logs to a logfile, if they're not already going somewhere" - l = logging.getLogger("paramiko") - if len(l.handlers) > 0: - return - l.setLevel(level) - f = open(filename, 'w') - lh = logging.StreamHandler(f) - lh.setFormatter(logging.Formatter('%(levelname)-.3s [%(asctime)s.%(msecs)03d] thr=%(_threadid)-3d %(name)s: %(message)s', - '%Y%m%d-%H:%M:%S')) - l.addHandler(lh) - -# make only one filter object, so it doesn't get applied more than once -class PFilter (object): - def filter(self, record): - record._threadid = get_thread_id() - return True -_pfilter = PFilter() - -def get_logger(name): - l = logging.getLogger(name) - l.addFilter(_pfilter) - return l - -def retry_on_signal(function): - """Retries function until it doesn't raise an EINTR error""" - while True: - try: - return function() - except EnvironmentError, e: - if e.errno != errno.EINTR: - raise - -class Counter (object): - """Stateful counter for CTR mode crypto""" - def __init__(self, nbits, initial_value=1L, overflow=0L): - self.blocksize = nbits / 8 - self.overflow = overflow - # start with value - 1 so we don't have to store intermediate values when counting - # could the iv be 0? - if initial_value == 0: - self.value = array.array('c', '\xFF' * self.blocksize) - else: - x = deflate_long(initial_value - 1, add_sign_padding=False) - self.value = array.array('c', '\x00' * (self.blocksize - len(x)) + x) - - def __call__(self): - """Increament the counter and return the new value""" - i = self.blocksize - 1 - while i > -1: - c = self.value[i] = chr((ord(self.value[i]) + 1) % 256) - if c != '\x00': - return self.value.tostring() - i -= 1 - # counter reset - x = deflate_long(self.overflow, add_sign_padding=False) - self.value = array.array('c', '\x00' * (self.blocksize - len(x)) + x) - return self.value.tostring() - - def new(cls, nbits, initial_value=1L, overflow=0L): - return cls(nbits, initial_value=initial_value, overflow=overflow) - new = classmethod(new) diff --git a/contrib/site-packages/paramiko/win_pageant.py b/contrib/site-packages/paramiko/win_pageant.py deleted file mode 100644 index de1cd64b..00000000 --- a/contrib/site-packages/paramiko/win_pageant.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright (C) 2005 John Arbash-Meinel -# Modified up by: Todd Whiteman -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - -""" -Functions for communicating with Pageant, the basic windows ssh agent program. -""" - -from __future__ import with_statement - -import struct -import threading -import array -import platform -import ctypes.wintypes - -from . import _winapi - -_AGENT_COPYDATA_ID = 0x804e50ba -_AGENT_MAX_MSGLEN = 8192 -# Note: The WM_COPYDATA value is pulled from win32con, as a workaround -# so we do not have to import this huge library just for this one variable. -win32con_WM_COPYDATA = 74 - - -def _get_pageant_window_object(): - return ctypes.windll.user32.FindWindowA('Pageant', 'Pageant') - - -def can_talk_to_agent(): - """ - Check to see if there is a "Pageant" agent we can talk to. - - This checks both if we have the required libraries (win32all or ctypes) - and if there is a Pageant currently running. - """ - return bool(_get_pageant_window_object()) - -ULONG_PTR = ctypes.c_uint64 if platform.architecture()[0] == '64bit' else ctypes.c_uint32 -class COPYDATASTRUCT(ctypes.Structure): - """ - ctypes implementation of - http://msdn.microsoft.com/en-us/library/windows/desktop/ms649010%28v=vs.85%29.aspx - """ - _fields_ = [ - ('num_data', ULONG_PTR), - ('data_size', ctypes.wintypes.DWORD), - ('data_loc', ctypes.c_void_p), - ] - -def _query_pageant(msg): - """ - Communication with the Pageant process is done through a shared - memory-mapped file. - """ - hwnd = _get_pageant_window_object() - if not hwnd: - # Raise a failure to connect exception, pageant isn't running anymore! - return None - - # create a name for the mmap - map_name = 'PageantRequest%08x' % threading.current_thread().ident - - pymap = _winapi.MemoryMap(map_name, _AGENT_MAX_MSGLEN, - _winapi.get_security_attributes_for_user(), - ) - with pymap: - pymap.write(msg) - # Create an array buffer containing the mapped filename - char_buffer = array.array("c", map_name + '\0') - char_buffer_address, char_buffer_size = char_buffer.buffer_info() - # Create a string to use for the SendMessage function call - cds = COPYDATASTRUCT(_AGENT_COPYDATA_ID, char_buffer_size, - char_buffer_address) - - response = ctypes.windll.user32.SendMessageA(hwnd, - win32con_WM_COPYDATA, ctypes.sizeof(cds), ctypes.byref(cds)) - - if response > 0: - pymap.seek(0) - datalen = pymap.read(4) - retlen = struct.unpack('>I', datalen)[0] - return datalen + pymap.read(retlen) - return None - -class PageantConnection (object): - """ - Mock "connection" to an agent which roughly approximates the behavior of - a unix local-domain socket (as used by Agent). Requests are sent to the - pageant daemon via special Windows magick, and responses are buffered back - for subsequent reads. - """ - - def __init__(self): - self._response = None - - def send(self, data): - self._response = _query_pageant(data) - - def recv(self, n): - if self._response is None: - return '' - ret = self._response[:n] - self._response = self._response[n:] - if self._response == '': - self._response = None - return ret - - def close(self): - pass From 30d4b9765d0becff4377a5f450dac9907068fb14 Mon Sep 17 00:00:00 2001 From: Nicholas Mills Date: Mon, 12 May 2014 17:10:20 -0400 Subject: [PATCH 14/50] Let XNI set the TCP window size (#18) --- src/xnet/xnet_end_to_end_init.c | 1 + src/xni/xni.h | 7 +++- src/xni/xni_tcp.c | 70 ++++++++++++++++++--------------- 3 files changed, 46 insertions(+), 32 deletions(-) diff --git a/src/xnet/xnet_end_to_end_init.c b/src/xnet/xnet_end_to_end_init.c index 934a8ba5..ad93020c 100644 --- a/src/xnet/xnet_end_to_end_init.c +++ b/src/xnet/xnet_end_to_end_init.c @@ -51,6 +51,7 @@ int32_t xint_e2e_xni_init(target_data_t *tdp) { if (xni_protocol_tcp == tdp->xni_pcl) rc = xni_allocate_tcp_control_block(num_threads, tdp->xni_tcp_congestion, + XNI_TCP_DEFAULT_WINDOW_SIZE, &tdp->xni_cb); #if HAVE_ENABLE_IB else if (xni_protocol_ib == tdp->xni_pcl) diff --git a/src/xni/xni.h b/src/xni/xni.h index ed65b4d5..0840716d 100644 --- a/src/xni/xni.h +++ b/src/xni/xni.h @@ -388,6 +388,7 @@ enum { XNI_TCP_DEFAULT_NUM_SOCKETS = 0, /*!< \brief Use the default number of sockets. */ }; extern const char *XNI_TCP_DEFAULT_CONGESTION; /*!< \brief Use the default TCP congestion avoidance algorithm. */ +enum { XNI_TCP_DEFAULT_WINDOW_SIZE = 0 }; /*!< \brief Use the operating system default TCP window size */ /*! \brief Create a control block for the TCP implementation. * * If \e num_sockets is #XNI_TCP_DEFAULT_NUM_SOCKETS then the number @@ -397,8 +398,12 @@ extern const char *XNI_TCP_DEFAULT_CONGESTION; /*!< \brief Use the default TCP * If \e congestion is #XNI_TCP_DEFAULT_CONGESTION then the system * default congestion avoidance algorithm will be used. * + * If \e window_size is #XNI_TCP_DEFAULT_WINDOW_SIZE then the + * operating system default TCP window size will be used. + * * \param num_sockets The number of TCP sockets to create per connection. * \param congestion the congestion control algorithm to use + * \param window_size the TCP window size * \param[out] control_block The newly allocated control block. * * \return #XNI_OK if the control block was successfully created. @@ -406,7 +411,7 @@ extern const char *XNI_TCP_DEFAULT_CONGESTION; /*!< \brief Use the default TCP * * \sa xni_free_tcp_control_block() */ -int xni_allocate_tcp_control_block(int num_sockets, const char *congestion, xni_control_block_t *control_block); + int xni_allocate_tcp_control_block(int num_sockets, const char *congestion, int window_size, xni_control_block_t *control_block); /*! \brief Free a TCP control block. * * It is forbidden to call this function more than once with the same diff --git a/src/xni/xni_tcp.c b/src/xni/xni_tcp.c index 0cdc9bfb..4bed53cd 100644 --- a/src/xni/xni_tcp.c +++ b/src/xni/xni_tcp.c @@ -30,6 +30,7 @@ static const size_t TCP_DATA_MESSAGE_HEADER_SIZE = 12; struct tcp_control_block { size_t num_sockets; char congestion[16]; + int window_size; }; struct tcp_context { @@ -78,7 +79,7 @@ struct tcp_target_buffer { }; -int xni_allocate_tcp_control_block(int num_sockets, const char *congestion, xni_control_block_t *cb_) +int xni_allocate_tcp_control_block(int num_sockets, const char *congestion, int window_size, xni_control_block_t *cb_) { struct tcp_control_block **cb = (struct tcp_control_block**)cb_; @@ -90,9 +91,14 @@ int xni_allocate_tcp_control_block(int num_sockets, const char *congestion, xni_ if (strlen(congestion) >= sizeof((*cb)->congestion)) return XNI_ERR; + // sanity check + if (window_size < 1 && window_size != XNI_TCP_DEFAULT_WINDOW_SIZE) + return XNI_ERR; + struct tcp_control_block *tmp = calloc(1, sizeof(*tmp)); tmp->num_sockets = num_sockets; strncpy(tmp->congestion, congestion, (sizeof(tmp->congestion) - 1)); + tmp->window_size = window_size; *cb = tmp; return XNI_OK; } @@ -226,26 +232,27 @@ static int tcp_accept_connection(xni_context_t ctx_, struct xni_endpoint* local, #endif // HAVE_DECL_TCP_CONGESTION } - //TODO: replace with a real value - optval = 131072; - int rc = setsockopt(servers[i], - SOL_SOCKET, - SO_SNDBUF, - &optval, - sizeof(optval)); - if (rc) { - perror("setsockopt"); - } + // optionally set the window size + if (ctx->control_block.window_size != XNI_TCP_DEFAULT_WINDOW_SIZE) { + optval = ctx->control_block.window_size; + int rc = setsockopt(servers[i], + SOL_SOCKET, + SO_SNDBUF, + &optval, + sizeof(optval)); + if (rc) { + perror("setsockopt"); + } - //TODO: replace with a real value - optval = 131072; - rc = setsockopt(servers[i], + optval = ctx->control_block.window_size; + rc = setsockopt(servers[i], SOL_SOCKET, SO_RCVBUF, &optval, sizeof(optval)); - if (rc) { - perror("setsockopt"); + if (rc) { + perror("setsockopt"); + } } struct sockaddr_in addr; @@ -346,26 +353,27 @@ static int tcp_connect(xni_context_t ctx_, struct xni_endpoint* remote, xni_conn #endif // HAVE_DECL_TCP_CONGESTION } - //TODO: replace with a real value - int optval = 131072; - int rc = setsockopt(servers[i].sockd, - SOL_SOCKET, - SO_SNDBUF, - &optval, - sizeof(optval)); - if (rc) { - perror("setsockopt"); - } + // optionally set the window size + if (ctx->control_block.window_size != XNI_TCP_DEFAULT_WINDOW_SIZE) { + int optval = ctx->control_block.window_size; + int rc = setsockopt(servers[i].sockd, + SOL_SOCKET, + SO_SNDBUF, + &optval, + sizeof(optval)); + if (rc) { + perror("setsockopt"); + } - //TODO: replace with a real value - optval = 131072; - rc = setsockopt(servers[i].sockd, + optval = ctx->control_block.window_size; + rc = setsockopt(servers[i].sockd, SOL_SOCKET, SO_RCVBUF, &optval, sizeof(optval)); - if (rc) { - perror("setsockopt"); + if (rc) { + perror("setsockopt"); + } } struct sockaddr_in addr; From 97ca693942a4b0b54cc05a7aca7dd2155987262b Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Wed, 28 May 2014 12:41:24 -0400 Subject: [PATCH 15/50] Use the correct default TCP window size (#18) Now XNI TCP will use the same window size as the legacy end-to-end TCP implementation. --- src/xnet/xnet_end_to_end_init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/xnet/xnet_end_to_end_init.c b/src/xnet/xnet_end_to_end_init.c index ad93020c..19d2388f 100644 --- a/src/xnet/xnet_end_to_end_init.c +++ b/src/xnet/xnet_end_to_end_init.c @@ -51,7 +51,7 @@ int32_t xint_e2e_xni_init(target_data_t *tdp) { if (xni_protocol_tcp == tdp->xni_pcl) rc = xni_allocate_tcp_control_block(num_threads, tdp->xni_tcp_congestion, - XNI_TCP_DEFAULT_WINDOW_SIZE, + tdp->td_planp->e2e_TCP_Win, &tdp->xni_cb); #if HAVE_ENABLE_IB else if (xni_protocol_ib == tdp->xni_pcl) From 7a849b41997cb4f086d5c4988e1df327c9c31281 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Thu, 29 May 2014 18:15:22 -0400 Subject: [PATCH 16/50] Clean up XNI memory registration functions Removes an extraneous parameter to xni_register_buffer(). Removes xni_unregister_buffer() because it was unused and unimplemented. --- src/base/worker_thread_init.c | 3 +-- src/xni/xni.c | 10 ++-------- src/xni/xni.h | 24 +++++------------------- src/xni/xni_ib.c | 14 +------------- src/xni/xni_internal.h | 3 +-- src/xni/xni_tcp.c | 9 +-------- tests/other/xni/basic-ib-test.c | 1 - tests/other/xni/basic-xni-test.c | 1 - 8 files changed, 11 insertions(+), 54 deletions(-) diff --git a/src/base/worker_thread_init.c b/src/base/worker_thread_init.c index b8ef60bb..6dfca7a7 100644 --- a/src/base/worker_thread_init.c +++ b/src/base/worker_thread_init.c @@ -150,8 +150,7 @@ xdd_worker_thread_init(worker_data_t *wdp) { memset(bufp, 0, 2*getpagesize()); /* Mark everything after the first page as reserved */ size_t reserve = getpagesize(); - xni_register_buffer(tdp->xni_ctx, bufp, wdp->wd_buf_size, reserve, - &wdp->wd_e2ep->xni_wd_buf); + xni_register_buffer(tdp->xni_ctx, bufp, wdp->wd_buf_size, reserve); xni_request_target_buffer(tdp->xni_ctx, &wdp->wd_e2ep->xni_wd_buf); bufp = xni_target_buffer_data(wdp->wd_e2ep->xni_wd_buf); } diff --git a/src/xni/xni.c b/src/xni/xni.c index c5835927..2f067c57 100644 --- a/src/xni/xni.c +++ b/src/xni/xni.c @@ -30,15 +30,9 @@ int xni_accept_connection(xni_context_t ctx, struct xni_endpoint* local, xni_con return ctx->protocol->accept_connection(ctx, local, conn); } -int xni_register_buffer(xni_context_t ctx, void* buf, size_t nbytes, size_t reserved, - xni_target_buffer_t* tbp) +int xni_register_buffer(xni_context_t ctx, void* buf, size_t nbytes, size_t reserved) { - return ctx->protocol->register_buffer(ctx, buf, nbytes, reserved, tbp); -} - -int xni_unregister_buffer(xni_context_t ctx, void* buf) -{ - return ctx->protocol->unregister_buffer(ctx, buf); + return ctx->protocol->register_buffer(ctx, buf, nbytes, reserved); } //TODO: local_endpoint diff --git a/src/xni/xni.h b/src/xni/xni.h index 0840716d..44dcb737 100644 --- a/src/xni/xni.h +++ b/src/xni/xni.h @@ -148,33 +148,19 @@ int xni_context_destroy(xni_context_t *context); /*! \brief Register memory with XNI. * * This function provides XNI drivers to perform optimizations based on - * the adress of the memory buffers in use. + * the address of the memory buffers in use. * - * \param[in,out] context The context to register the buffer with. + * \param context The context to register the buffer with. * \param[in] buf The memory buffer to register. - * \param[in] nbytes The total size of the buffer in bytes. - * \param[in] reserved The offset into the buffer at which the caller will + * \param nbytes The total size of the buffer in bytes. + * \param reserved The offset into the buffer at which the caller will * insert application data. Although this seems backwards, it ensures both * the caller and XNI can align data per their own requirements. - * \param[out] tb The xni target buffer to use for send/recvs. * * \return #XNI_OK if registration was successful. * \return #XNI_ERR if registration failed. - * - * \sa xni_unregister() - */ -int xni_register_buffer(xni_context_t context, void* buf, size_t nbytes, size_t reserved, xni_target_buffer_t* tb); -/*! \brief Free resources associated with registering memory with XNI. - * - * This function frees any resources used to register memory for use with - * XNI. - * - * \return #XNI_OK if the cleanup was successful. - * \return #XNI_ERR if the cleanup failed. - * - * \sa xni_register() */ -int xni_unregister_buffer(xni_context_t context, void* buf); +int xni_register_buffer(xni_context_t context, void* buf, size_t nbytes, size_t reserved); /*! \brief Wait for a connection from a remote process. * diff --git a/src/xni/xni_ib.c b/src/xni/xni_ib.c index 9c5567df..6cf47161 100644 --- a/src/xni/xni_ib.c +++ b/src/xni/xni_ib.c @@ -204,8 +204,7 @@ static int ib_context_destroy(xni_context_t *ctx_) return XNI_OK; } -static int ib_register_buffer(xni_context_t ctx_, void* buf, size_t nbytes, size_t reserved, - xni_target_buffer_t* xtb) +static int ib_register_buffer(xni_context_t ctx_, void* buf, size_t nbytes, size_t reserved) { struct ib_context* ctx = (struct ib_context*)ctx_; uintptr_t beginp = (uintptr_t)buf; @@ -241,19 +240,9 @@ static int ib_register_buffer(xni_context_t ctx_, void* buf, size_t nbytes, size ctx->num_registered++; pthread_mutex_unlock(&ctx->target_buffers_mutex); - // Set the outbound target buffer - *xtb= (xni_target_buffer_t)&tb; return XNI_OK; } -static int ib_unregister_buffer(xni_context_t ctx_, void* buf) -{ - struct ib_context* ctx = (struct ib_context*)ctx_; - pthread_mutex_lock(&ctx->target_buffers_mutex); - pthread_mutex_unlock(&ctx->target_buffers_mutex); - return XNI_OK; -} - static struct ib_credit_buffer **allocate_credit_buffers(struct ib_context *ctx, int nbuf) { struct ib_credit_buffer **credit_buffers = calloc((nbuf + 1), sizeof(*credit_buffers)); @@ -1062,7 +1051,6 @@ static struct xni_protocol protocol_ib = { .context_create = ib_context_create, .context_destroy = ib_context_destroy, .register_buffer = ib_register_buffer, - .unregister_buffer = ib_unregister_buffer, .accept_connection = ib_accept_connection, .connect = ib_connect, .close_connection = ib_close_connection, diff --git a/src/xni/xni_internal.h b/src/xni/xni_internal.h index fb7b8c81..d6bc5714 100644 --- a/src/xni/xni_internal.h +++ b/src/xni/xni_internal.h @@ -7,8 +7,7 @@ struct xni_protocol { int (*context_create)(xni_protocol_t, xni_control_block_t, xni_context_t*); int (*context_destroy)(xni_context_t*); - int (*register_buffer)(xni_context_t, void*, size_t, size_t, xni_target_buffer_t*); - int (*unregister_buffer)(xni_context_t, void*); + int (*register_buffer)(xni_context_t, void*, size_t, size_t); int (*accept_connection)(xni_context_t, struct xni_endpoint*, xni_connection_t*); int (*connect)(xni_context_t, struct xni_endpoint*, xni_connection_t*); diff --git a/src/xni/xni_tcp.c b/src/xni/xni_tcp.c index 4bed53cd..a8616396 100644 --- a/src/xni/xni_tcp.c +++ b/src/xni/xni_tcp.c @@ -147,7 +147,7 @@ static int tcp_context_destroy(xni_context_t *ctx_) return XNI_OK; } -static int tcp_register_buffer(xni_context_t ctx_, void* buf, size_t nbytes, size_t reserved, xni_target_buffer_t* tbp) { +static int tcp_register_buffer(xni_context_t ctx_, void* buf, size_t nbytes, size_t reserved) { struct tcp_context* ctx = (struct tcp_context*) ctx_; uintptr_t beginp = (uintptr_t)buf; uintptr_t datap = (uintptr_t)buf + (uintptr_t)(reserved); @@ -174,12 +174,6 @@ static int tcp_register_buffer(xni_context_t ctx_, void* buf, size_t nbytes, siz ctx->num_registered++; pthread_mutex_unlock(&ctx->buffer_mutex); - // Set the user's target buffer - *tbp = (xni_target_buffer_t)&(tb); - return XNI_OK; -} - -static int tcp_unregister_buffer(xni_context_t ctx, void* buf) { return XNI_OK; } @@ -673,7 +667,6 @@ static struct xni_protocol protocol_tcp = { .context_create = tcp_context_create, .context_destroy = tcp_context_destroy, .register_buffer = tcp_register_buffer, - .unregister_buffer = tcp_unregister_buffer, .accept_connection = tcp_accept_connection, .connect = tcp_connect, .close_connection = tcp_close_connection, diff --git a/tests/other/xni/basic-ib-test.c b/tests/other/xni/basic-ib-test.c index 6a32031f..7fe2e494 100644 --- a/tests/other/xni/basic-ib-test.c +++ b/tests/other/xni/basic-ib-test.c @@ -60,7 +60,6 @@ int start_server() xni_release_target_buffer(&xtb); // XNI cleanup stuff - xni_unregister_buffer(xni_ctx, buf); xni_finalize(); free(buf); #else diff --git a/tests/other/xni/basic-xni-test.c b/tests/other/xni/basic-xni-test.c index eb014a13..17e918fa 100644 --- a/tests/other/xni/basic-xni-test.c +++ b/tests/other/xni/basic-xni-test.c @@ -46,7 +46,6 @@ int start_server() xni_release_target_buffer(&xtb); // XNI cleanup stuff - xni_unregister_buffer(xni_ctx, buf); xni_finalize(); free(buf); return rc; From 936f92a20deea5e2fecbac809384549df6bd4dcb Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Fri, 30 May 2014 14:53:56 -0400 Subject: [PATCH 17/50] Add helper function to test for end-to-end --- src/common/xint_prototypes.h | 1 + src/xnet/xnet_end_to_end.c | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/src/common/xint_prototypes.h b/src/common/xint_prototypes.h index 65181cd2..6b4fb3b8 100644 --- a/src/common/xint_prototypes.h +++ b/src/common/xint_prototypes.h @@ -408,6 +408,7 @@ int32_t xint_e2e_dest_connect(target_data_t *tdp); int32_t xint_e2e_src_connect(target_data_t *tdp); int32_t xint_e2e_xni_send(worker_data_t *wdp); int32_t xint_e2e_xni_recv(worker_data_t *wdp); +int xint_is_e2e(const target_data_t *tdp); #endif /* * Local variables: diff --git a/src/xnet/xnet_end_to_end.c b/src/xnet/xnet_end_to_end.c index 2d63474a..13f018ae 100644 --- a/src/xnet/xnet_end_to_end.c +++ b/src/xnet/xnet_end_to_end.c @@ -287,6 +287,12 @@ int32_t xint_e2e_xni_recv(worker_data_t *wdp) { } /* end of xint_e2e_xni_recv() */ +int +xint_is_e2e(const target_data_t *tdp) +{ + return (1 == (TO_ENDTOEND & tdp->td_target_options)); +} + /* * Local variables: * indent-tabs-mode: t From 5345a7061a65b46ce337607dc1950fc97b901348 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Fri, 30 May 2014 14:55:38 -0400 Subject: [PATCH 18/50] Allocate I/O buffers in the target thread Previously the I/O buffers were allocated in the worker threads. This commit is a necessary first step for fixing end-to-end connections with XNI. Except that this commit breaks end-to-end transfers. --- src/base/io_buffers.c | 35 ++++--------- src/base/target_cleanup.c | 6 +++ src/base/target_init.c | 92 ++++++++++++++++++++++++----------- src/base/worker_thread_init.c | 44 +++-------------- src/common/target_data.c | 3 ++ src/common/xint_prototypes.h | 2 +- src/common/xint_td.h | 5 ++ 7 files changed, 95 insertions(+), 92 deletions(-) diff --git a/src/base/io_buffers.c b/src/base/io_buffers.c index 28748c7e..01d2a2b6 100644 --- a/src/base/io_buffers.c +++ b/src/base/io_buffers.c @@ -22,8 +22,7 @@ /*----------------------------------------------------------------------------*/ /* xdd_init_io_buffers() - set up the I/O buffers * This routine will allocate the memory used as the I/O buffer for a Worker - * Thread. The pointer to the buffer (wd_bufp) and the size of the buffer - * (wd_buf_size) are set in the Worker Data Struct. + * Thread. * * This routine will return the pointer to the buffer upon success. If for * some reason the buffer cannot be allocated then NULL is returned. @@ -34,32 +33,17 @@ * * The size of the buffer depends on whether it is being used for network * I/O as in an End-to-end operation. For End-to-End operations, the size - * of the buffer is 1 page larger than for non-End-to-End operations. + * of the buffer is 2 pages larger than for non-End-to-End operations. * * For normal (non-E2E operations) the buffer pointers are as follows: - * |<----------- wd_buf_size = N Pages ----------------->| + * |<------------------- N Pages ----------------------->| * +-----------------------------------------------------+ * | data buffer | * | transfer size (td_xfer_size) rounded up to N pages | - * |<-wd_bufp | - * |<-task_datap | * +-----------------------------------------------------+ - * - * For End-to-End operations, the buffer pointers are as follows: - * |<------------------- wd_buf_size = N+1 Pages ------------------------>| - * +----------------+-----------------------------------------------------+ - * |<----1 page---->| transfer size (td_xfer_size) rounded up to N pages | - * |<-wd_bufp |<-task_datap | - * | | E2E | E2E | - * | |<-Header->| data buffer | - * +-----*----------*-----------------------------------------------------+ - * ^ ^ - * ^ +-e2e_datap - * +-e2e_hdrp */ unsigned char * -xdd_init_io_buffers(worker_data_t *wdp) { - target_data_t *tdp; // Pointer to Target Data +xdd_init_io_buffers(target_data_t *tdp) { unsigned char *bufp; // Generic Buffer Pointer void *shmat_status; // Status of shmat() int buf_shmid; // Shared Memory ID @@ -70,9 +54,9 @@ xdd_init_io_buffers(worker_data_t *wdp) { LPVOID lpMsgBuf; /* Used for the error messages */ #endif - tdp = wdp->wd_tdp; - wdp->wd_bufp = NULL; - wdp->wd_buf_size = 0; + //TODO: move logic somewhere else -nlmills + //wdp->wd_bufp = NULL; + //wdp->wd_buf_size = 0; // Calaculate the number of pages needed for a buffer page_size = getpagesize(); @@ -171,8 +155,9 @@ xdd_init_io_buffers(worker_data_t *wdp) { /* Lock all pages in memory */ xdd_lock_memory(bufp, buffer_size, "RW BUFFER"); - wdp->wd_bufp = bufp; - wdp->wd_buf_size = buffer_size; + //TODO: move logic somewhere else -nlmills + //wdp->wd_bufp = bufp; + //wdp->wd_buf_size = buffer_size; return(bufp); } /* end of xdd_init_io_buffers() */ diff --git a/src/base/target_cleanup.c b/src/base/target_cleanup.c index 134f52ce..877811f9 100644 --- a/src/base/target_cleanup.c +++ b/src/base/target_cleanup.c @@ -48,5 +48,11 @@ xdd_target_thread_cleanup(target_data_t *tdp) { xni_close_connection(&tdp->td_e2ep->xni_td_conn); } + for (size_t i = 0; i < tdp->io_buffers_count; i++) { + free(tdp->io_buffers[i]); + } + free(tdp->io_buffers); + tdp->io_buffers = NULL; + tdp->io_buffers_count = 0; } // End of xdd_target_thread_cleanup() diff --git a/src/base/target_init.c b/src/base/target_init.c index d450fd03..4f2a21b3 100644 --- a/src/base/target_init.c +++ b/src/base/target_init.c @@ -15,6 +15,52 @@ */ #include "xint.h" +static size_t +xint_get_buffer_count(const target_data_t *tdp) +{ + size_t count = tdp->td_queue_depth; + + if (xint_is_e2e(tdp) && tdp->td_e2ep->e2e_address_table_port_count > 0) { + + count = tdp->td_e2ep->e2e_address_table_port_count; + } + + //TODO: is the buffer count always just tdp->td_queue_depth? + return count; +} + +static int +xint_allocate_io_buffers(target_data_t* tdp) +{ + const size_t buffer_count = xint_get_buffer_count(tdp); + + unsigned char **iobufs = calloc(buffer_count, sizeof(*iobufs)); + if (!iobufs) { + fprintf(xgp->errout,"%s: xint_allocate_io_buffers: Target %d: ERROR: Failed to allocate I/O buffer array.\n", + xgp->progname, + tdp->td_target_number); + return XDD_RC_BAD; + } + + for (size_t i = 0; i < buffer_count; i++) { + // allocate an I/O buffer + unsigned char *bufp = xdd_init_io_buffers(tdp); + if (bufp == NULL) { + fprintf(xgp->errout,"%s: xint_allocate_io_buffers: Target %d: ERROR: Failed to allocate I/O buffer.\n", + xgp->progname, + tdp->td_target_number); + return XDD_RC_BAD; + } + + // save allocated buffer into target's array of buffers + iobufs[i] = bufp; + } + + tdp->io_buffers = iobufs; + tdp->io_buffers_count = buffer_count; + return XDD_RC_GOOD; +} + /*----------------------------------------------------------------------------*/ /* xint_target_init() - Initialize a Target Thread * This subroutine will open the target file and perform some initial sanity @@ -32,10 +78,7 @@ */ int32_t xint_target_init(target_data_t *tdp) { - int32_t status; // Status of function calls -// nclk_t CurrentLocalTime; // Used the init the Global Clock -// nclk_t TimeDelta; // Used the init the Global Clock -// uint32_t sleepseconds; + int status; // Status of function calls #if (AIX) @@ -87,21 +130,6 @@ xint_target_init(target_data_t *tdp) { if (xgp->max_errors == 0) xgp->max_errors = tdp->td_target_ops; - /* If we are synchronizing to a Global Clock, let's synchronize - * here so that we all start at *roughly* the same time - */ - // FIXME - TOM review to see if this can go in plan_init -// if (xgp->gts_addr) { -// nclk_now(&CurrentLocalTime); -// while (CurrentLocalTime < xgp->ActualLocalStartTime) { -// TimeDelta = ((xgp->ActualLocalStartTime - CurrentLocalTime)/BILLION); -// if (TimeDelta > 2) { -// sleepseconds = TimeDelta - 2; -// sleep(sleepseconds); -// } -// nclk_now(&CurrentLocalTime); -// } -// } if (xgp->global_options & GO_TIMER_INFO) { fprintf(xgp->errout,"Starting now...\n"); fflush(xgp->errout); @@ -125,20 +153,21 @@ xint_target_init(target_data_t *tdp) { } // Special setup for an End-to-End operation - if (tdp->td_target_options & TO_ENDTOEND) { + if (xint_is_e2e(tdp)) { status = xdd_e2e_target_init(tdp); if (status) return(-1); } - // Start the WorkerThreads - status = xint_target_init_start_worker_threads(tdp); - if (status) - return(-1); + // Allocate I/O buffers and store pointers into tdp + status = xint_allocate_io_buffers(tdp); + if (status != XDD_RC_GOOD) { + return -1; + } - // If this is XNI, perform the connection here - xdd_plan_t *planp = tdp->td_planp; - if (PLAN_ENABLE_XNI & planp->plan_options) { + // if this is an end-to-end transfer than perform the connection(s) + if (xint_is_e2e(tdp)) { + //TODO: connect with the previously allocated buffers /* Perform the XNI accept/connect */ if (tdp->td_target_options & TO_E2E_DESTINATION) { status = xint_e2e_dest_connect(tdp); @@ -151,6 +180,11 @@ xint_target_init(target_data_t *tdp) { } } + // Start the WorkerThreads + status = xint_target_init_start_worker_threads(tdp); + if (status) + return(-1); + // Display the information for this target xdd_target_info(xgp->output, tdp); if (xgp->csvoutput) @@ -195,7 +229,7 @@ xint_target_init_barriers(target_data_t *tdp) { status += xdd_init_barrier(tdp->td_planp, &tdp->td_targetpass_worker_thread_passcomplete_barrier,tdp->td_queue_depth+1,tmpname); // The Target Pass E2E EOF Complete barrier - only initialized when an End-to-End operation is running - if (tdp->td_target_options & TO_ENDTOEND) { + if (xint_is_e2e(tdp)) { sprintf(tmpname,"T%04d>targetpass_worker_thread_eofcomplete_barrier",tdp->td_target_number); status += xdd_init_barrier(tdp->td_planp, &tdp->td_targetpass_worker_thread_eofcomplete_barrier,2,tmpname); } @@ -261,7 +295,7 @@ xint_target_init_start_worker_threads(target_data_t *tdp) { // Start a WorkerThread and wait for it to initialize wdp->wd_worker_number = q; - if (tdp->td_target_options & TO_ENDTOEND) { + if (xint_is_e2e(tdp)) { // Find an e2e entry that has a valid port count while (0 == tdp->td_e2ep->e2e_address_table[e2e_addr_index].port_count) { diff --git a/src/base/worker_thread_init.c b/src/base/worker_thread_init.c index 6dfca7a7..e17da62e 100644 --- a/src/base/worker_thread_init.c +++ b/src/base/worker_thread_init.c @@ -27,7 +27,6 @@ xdd_worker_thread_init(worker_data_t *wdp) { int32_t status; target_data_t *tdp; // Pointer to this worker_thread's target Data Struct char tmpname[XDD_BARRIER_MAX_NAME_LENGTH]; // Used to create unique names for the barriers - unsigned char *bufp; // Generic Buffer pointer #if defined(HAVE_CPUSET_T) && defined(HAVE_PTHREAD_ATTR_SETAFFINITY_NP) // BWS Print the cpuset @@ -104,43 +103,13 @@ xdd_worker_thread_init(worker_data_t *wdp) { wdp->wd_tot_wait.totw_is_released = 0; wdp->wd_tot_wait.totw_nextp = 0; - // Get the I/O buffer - // The xdd_init_io_buffers() routine will set wd_bufp and wd_buf_size to appropriate values. - // The size of the buffer depends on whether it is being used for network - // I/O as in an End-to-end operation. For End-to-End operations, the size - // of the buffer is at least 1 page larger than for non-End-to-End - // operations. - // - // For normal (non-E2E operations) the buffer pointers are as follows: - // |<----------- wd_buf_size = N Pages ----------------->| - // +-----------------------------------------------------+ - // | data buffer | - // | transfer size (td_xfer_size) rounded up to N pages | - // |<-wd_bufp | - // |<-task_datap | - // +-----------------------------------------------------+ - // For ease of reading this code, bufp == wdp->wd_bufp. - // - bufp = xdd_init_io_buffers(wdp); - if (bufp == NULL) { - fprintf(xgp->errout,"%s: xdd_worker_thread_init: Target %d WorkerThread %d: ERROR: Failed to allocate I/O buffer.\n", - xgp->progname, - tdp->td_target_number, - wdp->wd_worker_number); - return(-1); + //TODO: set the buffer pointers the correct way for e2e + if (!xint_is_e2e(tdp)) { + const int32_t workernum = wdp->wd_worker_number; + assert((uint32_t)workernum < tdp->io_buffers_count); + wdp->wd_task.task_datap = tdp->io_buffers[workernum]; } - // For End-to-End operations, the buffer pointers are as follows: - // |<------------------- wd_buf_size = N+1 Pages ------------------------>| - // +----------------+-----------------------------------------------------+ - // |<----1 page---->| transfer size (td_xfer_size) rounded up to N pages | - // |<-wd_bufp |<-task_datap | - // | | E2E | E2E | - // | |<-Header->| data buffer | - // +-----*----------*-----------------------------------------------------+ - // ^ ^ - // ^ +-e2e_datap - // +-e2e_hdrp - // +#if 0 if (tdp->td_target_options & TO_ENDTOEND) { /* If this e2e transfer is xni, register the buffer */ @@ -163,6 +132,7 @@ xdd_worker_thread_init(worker_data_t *wdp) { // For normal (non-E2E) operations the data portion is the entire buffer wdp->wd_task.task_datap = bufp; } +#endif // Set proper data pattern in Data buffer xdd_datapattern_buffer_init(wdp); diff --git a/src/common/target_data.c b/src/common/target_data.c index e91b7d07..00a89b30 100644 --- a/src/common/target_data.c +++ b/src/common/target_data.c @@ -105,6 +105,9 @@ xdd_init_new_target_data(target_data_t *tdp, int32_t n) { tdp->td_e2ep->e2e_address_table_next_entry=0; } + tdp->io_buffers = NULL; + tdp->io_buffers_count = 0; + tdp->xni_ibdevice = DEFAULT_IB_DEVICE; /* can be changed by '-ibdevice' CLO */ tdp->xni_tcp_congestion = XNI_TCP_DEFAULT_CONGESTION; /* can be changed by '-congestion' CLO */ diff --git a/src/common/xint_prototypes.h b/src/common/xint_prototypes.h index 6b4fb3b8..ad13965a 100644 --- a/src/common/xint_prototypes.h +++ b/src/common/xint_prototypes.h @@ -130,7 +130,7 @@ void xdd_interactive_show_trace(int32_t tokens, char *cmdline, uint32_t flags, x void xdd_interactive_show_barrier(int32_t tokens, char *cmdline, uint32_t flags, xdd_plan_t *planp); // io_buffers.c -unsigned char *xdd_init_io_buffers(worker_data_t *wdp); +unsigned char *xdd_init_io_buffers(target_data_t *wdp); // lockstep.c int32_t xdd_lockstep(target_data_t *p); diff --git a/src/common/xint_td.h b/src/common/xint_td.h index 8c7c19bc..3b321737 100644 --- a/src/common/xint_td.h +++ b/src/common/xint_td.h @@ -166,6 +166,11 @@ struct xint_target_data { #endif int32_t td_op_delay; // Number of seconds to delay between operations + // I/O buffers allocated and freed by the target thread but shared + // with either XNI or the workers + unsigned char **io_buffers; + size_t io_buffers_count; + /* XNI Networking components */ xni_protocol_t xni_pcl; xni_control_block_t xni_cb; From 6bc0d188049e04adae618d73ddaf3a2dd51deef5 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Fri, 30 May 2014 16:56:43 -0400 Subject: [PATCH 19/50] Remove unnecessary e2e header fields The whole e2e header will be moving to XNI soon. But before then I want to make the header as small as possible. --- src/common/debug.c | 4 +--- src/common/end_to_end.h | 4 +--- src/net/end_to_end.c | 3 --- src/net/end_to_end_init.c | 14 -------------- src/xnet/xnet_end_to_end.c | 1 - 5 files changed, 2 insertions(+), 24 deletions(-) diff --git a/src/common/debug.c b/src/common/debug.c index afa8381b..88990543 100644 --- a/src/common/debug.c +++ b/src/common/debug.c @@ -435,10 +435,8 @@ void xdd_show_e2e_header(xdd_e2e_header_t *e2ehp) { fprintf(stderr,"\nxdd_show_e2e_header:********* Start of E2E Header Data at 0x%p **********\n",e2ehp); fprintf(stderr,"\t\txdd_show_e2e_header: uint32_t e2eh_magic=0x%8x\n",e2ehp->e2eh_magic); // Magic Number - sanity check - fprintf(stderr,"\t\txdd_show_e2e_header: int32_t e2eh_worker_thread_number=%d\n",e2ehp->e2eh_worker_thread_number); // Sender's Worker Thread Number + fprintf(stderr,"\t\txdd_show_e2e_header: int32_t pad1\n"); fprintf(stderr,"\t\txdd_show_e2e_header: int64_t e2eh_sequence_number=%lld\n",(long long int)e2ehp->e2eh_sequence_number); // Sequence number of this operation - fprintf(stderr,"\t\txdd_show_e2e_header: nclk_t e2eh_send_time=%lld\n",(unsigned long long int)e2ehp->e2eh_send_time); // Time this packet was sent in global nano seconds - fprintf(stderr,"\t\txdd_show_e2e_header: nclk_t e2eh_recv_time=%lld\n",(unsigned long long int)e2ehp->e2eh_recv_time); // Time this packet was received in global nano seconds fprintf(stderr,"\t\txdd_show_e2e_header: int64_t e2eh_byte_offset=%lld\n",(long long int)e2ehp->e2eh_byte_offset); // Offset relative to the beginning of the file of where this data belongs fprintf(stderr,"\t\txdd_show_e2e_header: int64_t e2eh_data_length=%lld\n",(long long int)e2ehp->e2eh_data_length); // Length of the user data in bytes for this operation fprintf(stderr,"\txdd_show_e2e_header:********* End of E2E Header Data at 0x%p **********\n",e2ehp); diff --git a/src/common/end_to_end.h b/src/common/end_to_end.h index 1ec36241..202cd2f9 100644 --- a/src/common/end_to_end.h +++ b/src/common/end_to_end.h @@ -33,10 +33,8 @@ #define E2E_ADDRESS_TABLE_ENTRIES 16 struct xdd_e2e_header { uint32_t e2eh_magic; // Magic Number - sanity check - int32_t e2eh_worker_thread_number; // Sender's Worker Thread Number + int32_t pad1; int64_t e2eh_sequence_number; // Sequence number of this operation - nclk_t e2eh_send_time; // Time this packet was sent in global nano seconds - nclk_t e2eh_recv_time; // Time this packet was received in global nano seconds int64_t e2eh_byte_offset; // Offset relative to the beginning of the file of where this data belongs int64_t e2eh_data_length; // Length of the user data in bytes for this operation }; diff --git a/src/net/end_to_end.c b/src/net/end_to_end.c index 71ad7fed..716db644 100644 --- a/src/net/end_to_end.c +++ b/src/net/end_to_end.c @@ -71,7 +71,6 @@ xdd_e2e_src_send(worker_data_t *wdp) { if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_src_send: Target: %d: Worker: %d: ENTER: e2ep=%p: e2ehp=%p: e2e_datap=%p\n",(long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, e2ep, e2ehp, e2ep->e2e_datap); // The "task" data structure contains variables relevant to the file-read operation - e2ehp->e2eh_worker_thread_number = wdp->wd_worker_number; e2ehp->e2eh_sequence_number = wdp->wd_task.task_op_number; e2ehp->e2eh_byte_offset = wdp->wd_task.task_byte_offset; e2ehp->e2eh_data_length = wdp->wd_task.task_xfer_size; @@ -519,7 +518,6 @@ if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e e2ep->e2e_sr_time = (wdp->wd_counters.tc_current_net_end_time - wdp->wd_counters.tc_current_net_start_time); } - e2ehp->e2eh_recv_time = wdp->wd_counters.tc_current_net_end_time; // This needs to be the net_end_time from this side of the operation // If time stamping is on then we need to reset these values if ((tdp->td_ts_table.ts_options & (TS_ON|TS_TRIGGERED))) { @@ -578,7 +576,6 @@ if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e max_xfer = MAXMIT_TCP; nclk_now(&wdp->wd_counters.tc_current_net_start_time); - e2ehp->e2eh_worker_thread_number = wdp->wd_worker_number; e2ehp->e2eh_sequence_number = (wdp->wd_task.task_op_number + wdp->wd_worker_number); // This is an EOF packet header e2ehp->e2eh_byte_offset = -1; // NA e2ehp->e2eh_data_length = 0; // NA - no data being sent other than the header diff --git a/src/net/end_to_end_init.c b/src/net/end_to_end_init.c index e0babe0d..fcf15c10 100644 --- a/src/net/end_to_end_init.c +++ b/src/net/end_to_end_init.c @@ -153,21 +153,7 @@ xdd_e2e_src_init(worker_data_t *wdp) { e2ep->e2e_header_size = (int)(sizeof(xdd_e2e_header_t)); // Init the message header - // For End-to-End operations, the buffer pointers are as follows: - // +----------------+-----------------------------------------------------+ - // |<----1 page---->| transfer size (td_xfer_size) rounded up to N pages | - // |<-task_bufp |<-task_datap | - // | | | | - // | |<-Header->| E2E data buffer | - // +-----*----------*-----------------------------------------------------+ - // ^ ^ - // ^ +-e2e_datap - // +-e2e_hdrp - // - e2ep->e2e_hdrp->e2eh_worker_thread_number = 0; e2ep->e2e_hdrp->e2eh_sequence_number = 0; - e2ep->e2e_hdrp->e2eh_send_time = 0; - e2ep->e2e_hdrp->e2eh_recv_time = 0; e2ep->e2e_hdrp->e2eh_byte_offset = 0; e2ep->e2e_hdrp->e2eh_data_length = 0; diff --git a/src/xnet/xnet_end_to_end.c b/src/xnet/xnet_end_to_end.c index 13f018ae..6b80a1fe 100644 --- a/src/xnet/xnet_end_to_end.c +++ b/src/xnet/xnet_end_to_end.c @@ -139,7 +139,6 @@ int32_t xint_e2e_xni_send(worker_data_t *wdp) { } /* Construct the e2e header */ - e2ehp->e2eh_worker_thread_number = wdp->wd_worker_number; e2ehp->e2eh_sequence_number = wdp->wd_task.task_op_number; e2ehp->e2eh_byte_offset = wdp->wd_task.task_byte_offset; e2ehp->e2eh_data_length = wdp->wd_task.task_xfer_size; From 71b3265ed49b5c8b8ed92c0e8962130a855326f4 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Fri, 30 May 2014 18:22:08 -0400 Subject: [PATCH 20/50] Make XNI use network byte order for all messages --- src/xni/xni_ib.c | 21 ++++++++++++--------- src/xni/xni_internal.h | 30 ++++++++++++++++++++++++++++++ src/xni/xni_tcp.c | 6 ++++-- 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/src/xni/xni_ib.c b/src/xni/xni_ib.c index 6cf47161..d30dfd7a 100644 --- a/src/xni/xni_ib.c +++ b/src/xni/xni_ib.c @@ -421,9 +421,8 @@ static int send_credits(struct ib_connection *conn, int ncredits) // // encode and send the credit message // - //TODO: NBO? memcpy(cb->msgbuf, CREDIT_MESSAGE_TAG, TAG_LENGTH); - uint32_t tmp32 = (uint32_t)ncredits; + uint32_t tmp32 = htonl((uint32_t)ncredits); memcpy(cb->msgbuf+TAG_LENGTH, &tmp32, 4); struct ibv_sge sge; @@ -478,8 +477,8 @@ static int consume_credit(struct ib_connection *conn) if (completed > 0) { struct ib_credit_buffer *cb = (struct ib_credit_buffer*)wc.wr_id; uint32_t tmp32 = 0; - //TODO: NBO? memcpy(&tmp32, cb->msgbuf+TAG_LENGTH, 4); + tmp32 = ntohl(tmp32); //TODO: check for deadlock if receive can't be posted post_receive(conn->queue_pair, cb->memory_region, cb->msgbuf, IB_CREDIT_MESSAGE_SIZE, (uintptr_t)cb); @@ -610,7 +609,9 @@ static int ib_accept_connection(xni_context_t ctx_, struct xni_endpoint* local, goto error_out; memcpy(&remote_qpnum, msgbuf, 4); + remote_qpnum = ntohl(remote_qpnum); memcpy(&remote_lid, msgbuf+4, 2); + remote_lid = ntohs(remote_lid); #ifdef XNI_TRACE printf("Remote QPN=%u, LID=%u\n", @@ -618,13 +619,13 @@ static int ib_accept_connection(xni_context_t ctx_, struct xni_endpoint* local, (unsigned int)remote_lid); #endif // XNI_TRACE - uint32_t tmp32 = qp->qp_num; + uint32_t tmp32 = htonl(qp->qp_num); memcpy(msgbuf, &tmp32, 4); struct ibv_port_attr portattr; memset(&portattr, 0, sizeof(portattr)); if (ibv_query_port(ctx->verbs_context, 1, &portattr)) goto error_out; - uint16_t tmp16 = portattr.lid; + uint16_t tmp16 = htons(portattr.lid); memcpy(msgbuf+4, &tmp16, 2); #ifdef XNI_TRACE @@ -763,13 +764,13 @@ static int ib_connect(xni_context_t ctx_, struct xni_endpoint* remote, xni_conne uint32_t remote_qpnum; uint16_t remote_lid; - uint32_t tmp32 = qp->qp_num; + uint32_t tmp32 = htonl(qp->qp_num); memcpy(msgbuf, &tmp32, 4); struct ibv_port_attr portattr; memset(&portattr, 0, sizeof(portattr)); if (ibv_query_port(ctx->verbs_context, 1, &portattr)) goto error_out; - uint16_t tmp16 = portattr.lid; + uint16_t tmp16 = htons(portattr.lid); memcpy(msgbuf+4, &tmp16, 2); #ifdef XNI_TRACE @@ -787,7 +788,9 @@ static int ib_connect(xni_context_t ctx_, struct xni_endpoint* remote, xni_conne client = -1; memcpy(&remote_qpnum, msgbuf, 4); + remote_qpnum = ntohl(remote_qpnum); memcpy(&remote_lid, msgbuf+4, 2); + remote_lid = ntohs(remote_lid); #ifdef XNI_TRACE printf("Remote QPN=%u, LID=%u\n", @@ -915,9 +918,8 @@ static int ib_send_target_buffer(xni_connection_t conn_, xni_target_buffer_t *ta goto free_out; // encode the message - //TODO: NBO? memcpy(tb->header, DATA_MESSAGE_TAG, TAG_LENGTH); - uint64_t tmp64 = tb->target_offset; + uint64_t tmp64 = ntohll(tb->target_offset); memcpy(((char*)tb->header)+TAG_LENGTH, &tmp64, 8); // send the message @@ -1000,6 +1002,7 @@ static int ib_receive_target_buffer(xni_connection_t conn_, xni_target_buffer_t tb = (struct ib_target_buffer*)wc.wr_id; if (memcmp(tb->header, DATA_MESSAGE_TAG, TAG_LENGTH) == 0) { memcpy(&tb->target_offset, ((char*)tb->header)+TAG_LENGTH, 8); + tb->target_offset = ntohll(tb->target_offset); tb->data_length = wc.byte_len - (int)((char*)tb->data - (char*)tb->header); } else if (memcmp(tb->header, EOF_MESSAGE_TAG, TAG_LENGTH) == 0) conn->eof = 1; diff --git a/src/xni/xni_internal.h b/src/xni/xni_internal.h index d6bc5714..6d5cbd0a 100644 --- a/src/xni/xni_internal.h +++ b/src/xni/xni_internal.h @@ -2,6 +2,9 @@ #define XDD_XNI_INTERNAL_H +#include // for ntohl() + + struct xni_protocol { const char *name; int (*context_create)(xni_protocol_t, xni_control_block_t, xni_context_t*); @@ -34,6 +37,33 @@ struct xni_target_buffer { int data_length; }; + +/** + * Convert from network byte-order (big endian) to host order + */ +static inline uint64_t ntohll(uint64_t value) +{ + int endian_test = 1; + + // Determine if host order is little endian + if (endian_test == *((char*)(&endian_test))) { + // Swap the bytes + uint32_t low = ntohl((uint32_t)(value & 0xFFFFFFFFLL)); + uint32_t high = ntohl((uint32_t)(value >> 32)); + value = ((uint64_t)(low) << 32) | (uint64_t)(high); + } + return value; +} + +/** + * Convert from host byte-order to network byte-order (big endian) + */ +static inline uint64_t htonll(uint64_t value) +{ + // Re-use the htonll implementation to swap the bytes + return htonll(value); +} + #endif // XDD_XNI_INTERNAL_H /* diff --git a/src/xni/xni_tcp.c b/src/xni/xni_tcp.c index a8616396..85334ae6 100644 --- a/src/xni/xni_tcp.c +++ b/src/xni/xni_tcp.c @@ -470,9 +470,9 @@ static int tcp_send_target_buffer(xni_connection_t conn_, xni_target_buffer_t *t return XNI_ERR; // encode the message header - uint64_t tmp64 = tb->target_offset; + uint64_t tmp64 = htonll(tb->target_offset); memcpy(tb->header, &tmp64, 8); - uint32_t tmp32 = tb->data_length; + uint32_t tmp32 = htonl(tb->data_length); memcpy(((char*)tb->header)+8, &tmp32, 4); // locate a free socket @@ -599,8 +599,10 @@ static int tcp_receive_target_buffer(xni_connection_t conn_, xni_target_buffer_t uint64_t target_offset; memcpy(&target_offset, recvbuf, 8); + target_offset = ntohll(target_offset); uint32_t data_length; memcpy(&data_length, recvbuf+8, 4); + data_length = ntohl(data_length); recvbuf = (char*)tb->data; total = data_length; From 67c7371d24eeee37b8fe23516eef93a1c956aed7 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Mon, 2 Jun 2014 08:21:49 -0400 Subject: [PATCH 21/50] Use the correct byte conversion function The wire protocol won't change (htonll(x) == ntohll(x)), but of course it's always better to use the correct function for documentation purposes. --- src/xni/xni_ib.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/xni/xni_ib.c b/src/xni/xni_ib.c index d30dfd7a..17bd008b 100644 --- a/src/xni/xni_ib.c +++ b/src/xni/xni_ib.c @@ -919,7 +919,7 @@ static int ib_send_target_buffer(xni_connection_t conn_, xni_target_buffer_t *ta // encode the message memcpy(tb->header, DATA_MESSAGE_TAG, TAG_LENGTH); - uint64_t tmp64 = ntohll(tb->target_offset); + uint64_t tmp64 = htonll(tb->target_offset); memcpy(((char*)tb->header)+TAG_LENGTH, &tmp64, 8); // send the message From ac5c0f0a84f91682f950759d1069d90cb6a4d1c4 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Mon, 2 Jun 2014 08:34:25 -0400 Subject: [PATCH 22/50] Send the operation sequence number with XNI The goal is to remove the xdd_e2e_header by sending the important information in the XNI message header. --- src/xni/xni.c | 10 ++++++++++ src/xni/xni.h | 20 +++++++++++++++++--- src/xni/xni_ib.c | 15 +++++++++++---- src/xni/xni_internal.h | 1 + src/xni/xni_tcp.c | 22 ++++++++++++++++------ 5 files changed, 55 insertions(+), 13 deletions(-) diff --git a/src/xni/xni.c b/src/xni/xni.c index 2f067c57..291c60b6 100644 --- a/src/xni/xni.c +++ b/src/xni/xni.c @@ -91,3 +91,13 @@ void xni_target_buffer_set_data_length(int length, xni_target_buffer_t tb) { tb->data_length = length; } + +int64_t xni_target_buffer_sequence_number(xni_target_buffer_t tb) +{ + return tb->sequence_number; +} + +void xni_target_buffer_set_sequence_number(int64_t seq, xni_target_buffer_t tb) +{ + tb->sequence_number = seq; +} diff --git a/src/xni/xni.h b/src/xni/xni.h index 44dcb737..d47840e1 100644 --- a/src/xni/xni.h +++ b/src/xni/xni.h @@ -307,9 +307,6 @@ int xni_receive_target_buffer(xni_connection_t connection, xni_target_buffer_t * int xni_release_target_buffer(xni_target_buffer_t *buffer); /*! \brief Get a target buffer's data pointer. - * - * The data pointer will point to a block of memory aligned on a - * 512-byte boundary. * * \param buffer The buffer to inspect. * @@ -362,6 +359,23 @@ int xni_target_buffer_data_length(xni_target_buffer_t buffer); * \sa xni_target_buffer_data_length() */ void xni_target_buffer_set_data_length(int length, xni_target_buffer_t buffer); +/*! \brief Get the operation sequence number. + * + * \param buffer The buffer to inspect. + * + * \return The operation sequence number. + * + * \sa xni_target_buffer_set_sequence_number() + */ +int64_t xni_target_buffer_sequence_number(xni_target_buffer_t buffer); +/*! \brief Set the operation sequence number. + * + * \param sequence_number The operation sequence number. + * \param buffer The buffer to modify. + * + * \sa xni_target_buffer_sequence_number() + */ +void xni_target_buffer_set_sequence_number(int64_t sequence_number, xni_target_buffer_t buffer); /*! @} */ diff --git a/src/xni/xni_ib.c b/src/xni/xni_ib.c index 17bd008b..4cab8d74 100644 --- a/src/xni/xni_ib.c +++ b/src/xni/xni_ib.c @@ -21,7 +21,7 @@ #include "xni_internal.h" //#define XNI_TRACE 1 -#define PROTOCOL_NAME "ib-nlmills-20120809" +#define PROTOCOL_NAME "ib-nlmills-20140602" #define ALIGN(val,align) (((val)+(align)-1UL) & ~((align)-1UL)) @@ -76,7 +76,9 @@ struct ib_connection { uint16_t remote_lid; }; -#define IB_DATA_MESSAGE_HEADER_SIZE 12 // = tag(4) + target_offset(8) +#define IB_DATA_MESSAGE_HEADER_SIZE 20 // = tag(4) + + // sequence_number(8) + + // target_offset(8) enum send_state { QUEUED, SENT, @@ -86,6 +88,7 @@ struct ib_target_buffer { // inherited from xni_target_buffer struct ib_context *context; void *data; + int64_t sequence_number; size_t target_offset; int data_length; @@ -919,8 +922,10 @@ static int ib_send_target_buffer(xni_connection_t conn_, xni_target_buffer_t *ta // encode the message memcpy(tb->header, DATA_MESSAGE_TAG, TAG_LENGTH); - uint64_t tmp64 = htonll(tb->target_offset); + uint64_t tmp64 = htonll(tb->sequence_number); memcpy(((char*)tb->header)+TAG_LENGTH, &tmp64, 8); + tmp64 = htonll(tb->target_offset); + memcpy(((char*)tb->header)+TAG_LENGTH+8, &tmp64, 8); // send the message struct ibv_sge sge; @@ -1001,7 +1006,9 @@ static int ib_receive_target_buffer(xni_connection_t conn_, xni_target_buffer_t // decode the message tb = (struct ib_target_buffer*)wc.wr_id; if (memcmp(tb->header, DATA_MESSAGE_TAG, TAG_LENGTH) == 0) { - memcpy(&tb->target_offset, ((char*)tb->header)+TAG_LENGTH, 8); + memcpy(&tb->sequence_number, ((char*)tb->header)+TAG_LENGTH, 8); + tb->sequence_number = ntohll(tb->sequence_number); + memcpy(&tb->target_offset, ((char*)tb->header)+TAG_LENGTH+8, 8); tb->target_offset = ntohll(tb->target_offset); tb->data_length = wc.byte_len - (int)((char*)tb->data - (char*)tb->header); } else if (memcmp(tb->header, EOF_MESSAGE_TAG, TAG_LENGTH) == 0) diff --git a/src/xni/xni_internal.h b/src/xni/xni_internal.h index 6d5cbd0a..d91eb895 100644 --- a/src/xni/xni_internal.h +++ b/src/xni/xni_internal.h @@ -33,6 +33,7 @@ struct xni_connection { struct xni_target_buffer { struct xni_context *context; void *data; + int64_t sequence_number; size_t target_offset; int data_length; }; diff --git a/src/xni/xni_tcp.c b/src/xni/xni_tcp.c index 85334ae6..1ddfb5eb 100644 --- a/src/xni/xni_tcp.c +++ b/src/xni/xni_tcp.c @@ -20,12 +20,14 @@ #include "xni_internal.h" -#define PROTOCOL_NAME "tcp-nlmills-20120809" +#define PROTOCOL_NAME "tcp-nlmills-20140602" #define ALIGN(val,align) (((val)+(align)-1UL) & ~((align)-1UL)) const char *XNI_TCP_DEFAULT_CONGESTION = ""; -static const size_t TCP_DATA_MESSAGE_HEADER_SIZE = 12; +static const size_t TCP_DATA_MESSAGE_HEADER_SIZE = 20; // = sequence_number(8) + + // target_offset(8) + + // data_length(4) struct tcp_control_block { size_t num_sockets; @@ -70,6 +72,7 @@ struct tcp_target_buffer { // inherited from xni_target_buffer struct tcp_context *context; void *data; + int64_t sequence_number; size_t target_offset; int data_length; @@ -470,10 +473,12 @@ static int tcp_send_target_buffer(xni_connection_t conn_, xni_target_buffer_t *t return XNI_ERR; // encode the message header - uint64_t tmp64 = htonll(tb->target_offset); + uint64_t tmp64 = htonll(tb->sequence_number); memcpy(tb->header, &tmp64, 8); + tmp64 = htonll(tb->target_offset); + memcpy(((char*)tb->header)+8, &tmp64, 8); uint32_t tmp32 = htonl(tb->data_length); - memcpy(((char*)tb->header)+8, &tmp32, 4); + memcpy(((char*)tb->header)+16, &tmp32, 4); // locate a free socket struct tcp_socket *socket = NULL; @@ -597,11 +602,15 @@ static int tcp_receive_target_buffer(xni_connection_t conn_, xni_target_buffer_t received += cnt; } + // decode the header + uint64_t sequence_number; + memcpy(&sequence_number, recvbuf, 8); + sequence_number = ntohll(sequence_number); uint64_t target_offset; - memcpy(&target_offset, recvbuf, 8); + memcpy(&target_offset, recvbuf+8, 8); target_offset = ntohll(target_offset); uint32_t data_length; - memcpy(&data_length, recvbuf+8, 4); + memcpy(&data_length, recvbuf+16, 4); data_length = ntohl(data_length); recvbuf = (char*)tb->data; @@ -618,6 +627,7 @@ static int tcp_receive_target_buffer(xni_connection_t conn_, xni_target_buffer_t } //TODO: sanity checks (e.g. tb->connection) + tb->sequence_number = sequence_number; tb->target_offset = target_offset; tb->data_length = (int)data_length; From 61ac0f7d0d4f79fa81a1ede21b6a741d8d22975c Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Mon, 2 Jun 2014 15:17:58 -0400 Subject: [PATCH 23/50] Fix infinite recursion bug --- src/common/net_utils.h | 4 ++-- src/xni/xni_internal.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/common/net_utils.h b/src/common/net_utils.h index 5527bbcd..edf0bb70 100644 --- a/src/common/net_utils.h +++ b/src/common/net_utils.h @@ -43,8 +43,8 @@ static inline uint64_t ntohll(uint64_t value) */ static inline uint64_t htonll(uint64_t value) { - // Re-use the htonll implementation to swap the bytes - return htonll(value); + // Re-use the ntohll implementation to swap the bytes + return ntohll(value); } #endif /* NET_UTILS_H */ diff --git a/src/xni/xni_internal.h b/src/xni/xni_internal.h index d91eb895..8be393ef 100644 --- a/src/xni/xni_internal.h +++ b/src/xni/xni_internal.h @@ -61,8 +61,8 @@ static inline uint64_t ntohll(uint64_t value) */ static inline uint64_t htonll(uint64_t value) { - // Re-use the htonll implementation to swap the bytes - return htonll(value); + // Re-use the ntohll implementation to swap the bytes + return ntohll(value); } #endif // XDD_XNI_INTERNAL_H From 12e912a3df40e7df30f78df2876981a839b87ddd Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Mon, 2 Jun 2014 15:57:55 -0400 Subject: [PATCH 24/50] XNI: store target buffers in the connection Target buffers are now allocated by the client but registered by xni_accept_connection() or xni_connect(). Now that target buffers have moved from the xni_context to the xni_connection we should be able to easily open multiple connections. With multiple connections we should be able to support multiple NICs. --- src/base/io_buffers.c | 10 +- src/base/worker_thread_init.c | 56 ++++---- src/base/worker_thread_ttd_after_io_op.c | 30 ++-- src/common/end_to_end.h | 3 - src/common/target_data.c | 1 + src/common/xint_td.h | 3 + src/xnet/xnet_end_to_end.c | 35 +++-- src/xni/xni.c | 19 +-- src/xni/xni.h | 49 +++---- src/xni/xni_ib.c | 173 +++++++++++++---------- src/xni/xni_internal.h | 10 +- src/xni/xni_tcp.c | 153 +++++++++++--------- 12 files changed, 290 insertions(+), 252 deletions(-) diff --git a/src/base/io_buffers.c b/src/base/io_buffers.c index 01d2a2b6..287a52d8 100644 --- a/src/base/io_buffers.c +++ b/src/base/io_buffers.c @@ -54,10 +54,6 @@ xdd_init_io_buffers(target_data_t *tdp) { LPVOID lpMsgBuf; /* Used for the error messages */ #endif - //TODO: move logic somewhere else -nlmills - //wdp->wd_bufp = NULL; - //wdp->wd_buf_size = 0; - // Calaculate the number of pages needed for a buffer page_size = getpagesize(); pages = tdp->td_xfer_size / page_size; @@ -155,9 +151,9 @@ xdd_init_io_buffers(target_data_t *tdp) { /* Lock all pages in memory */ xdd_lock_memory(bufp, buffer_size, "RW BUFFER"); - //TODO: move logic somewhere else -nlmills - //wdp->wd_bufp = bufp; - //wdp->wd_buf_size = buffer_size; + // the size of each buffer must be the same + assert(0 == tdp->io_buffer_size || (size_t)buffer_size == tdp->io_buffer_size); + tdp->io_buffer_size = buffer_size; return(bufp); } /* end of xdd_init_io_buffers() */ diff --git a/src/base/worker_thread_init.c b/src/base/worker_thread_init.c index e17da62e..afa5b2ba 100644 --- a/src/base/worker_thread_init.c +++ b/src/base/worker_thread_init.c @@ -103,39 +103,41 @@ xdd_worker_thread_init(worker_data_t *wdp) { wdp->wd_tot_wait.totw_is_released = 0; wdp->wd_tot_wait.totw_nextp = 0; - //TODO: set the buffer pointers the correct way for e2e - if (!xint_is_e2e(tdp)) { + // Set up I/O buffer pointers + if (xint_is_e2e(tdp)) { + if (tdp->td_target_options & TO_E2E_DESTINATION) { + // Buffer for destination is set after a receive + wdp->wd_e2ep->xni_wd_buf = NULL; + wdp->wd_task.task_datap = NULL; + wdp->wd_e2ep->e2e_datap = NULL; + wdp->wd_e2ep->e2e_hdrp = NULL; + } else { + //TODO: I'd really like to move all of these buffer + // request calls to something like ttd_before_io_op, but I + // can't right now because later on in this function + // xdd_datapattern_buffer_init() relies on the buffer + // being present. -nlmills + + // Request an I/O buffer from XNI + xni_request_target_buffer(tdp->td_e2ep->xni_td_conn, &wdp->wd_e2ep->xni_wd_buf); + unsigned char *bufp = xni_target_buffer_data(wdp->wd_e2ep->xni_wd_buf); + memset(bufp, 0, 2*getpagesize()); + // Use the first page for the E2E header + wdp->wd_task.task_datap = bufp + getpagesize(); + wdp->wd_e2ep->e2e_datap = wdp->wd_task.task_datap; + wdp->wd_e2ep->e2e_hdrp = (xdd_e2e_header_t *)(bufp + (getpagesize() - sizeof(xdd_e2e_header_t))); + } + } else { + // For non-E2E operations the data portion is the entire buffer const int32_t workernum = wdp->wd_worker_number; assert((uint32_t)workernum < tdp->io_buffers_count); wdp->wd_task.task_datap = tdp->io_buffers[workernum]; } -#if 0 - if (tdp->td_target_options & TO_ENDTOEND) { - /* If this e2e transfer is xni, register the buffer */ - xdd_plan_t *planp = wdp->wd_tdp->td_planp; - if (PLAN_ENABLE_XNI & planp->plan_options) { - /* Clear the two sparsely used pages for header data */ - memset(bufp, 0, 2*getpagesize()); - /* Mark everything after the first page as reserved */ - size_t reserve = getpagesize(); - xni_register_buffer(tdp->xni_ctx, bufp, wdp->wd_buf_size, reserve); - xni_request_target_buffer(tdp->xni_ctx, &wdp->wd_e2ep->xni_wd_buf); - bufp = xni_target_buffer_data(wdp->wd_e2ep->xni_wd_buf); - } - - /* Use the first page for the E2E header */ - wdp->wd_task.task_datap = bufp + getpagesize(); - wdp->wd_e2ep->e2e_datap = wdp->wd_task.task_datap; - wdp->wd_e2ep->e2e_hdrp = (xdd_e2e_header_t *)(bufp + (getpagesize() - sizeof(xdd_e2e_header_t))); - } else { - // For normal (non-E2E) operations the data portion is the entire buffer - wdp->wd_task.task_datap = bufp; + // Set the buffer data pattern for non-E2E operations or E2E sources + if (!xint_is_e2e(tdp) || !(tdp->td_target_options & TO_E2E_DESTINATION)) { + xdd_datapattern_buffer_init(wdp); } -#endif - - // Set proper data pattern in Data buffer - xdd_datapattern_buffer_init(wdp); // Init the WorkerThread-TargetPass WAIT Barrier for this WorkerThread sprintf(tmpname,"T%04d:W%04d>worker_thread_targetpass_wait_barrier",tdp->td_target_number,wdp->wd_worker_number); diff --git a/src/base/worker_thread_ttd_after_io_op.c b/src/base/worker_thread_ttd_after_io_op.c index 72729a97..d7ca05ff 100644 --- a/src/base/worker_thread_ttd_after_io_op.c +++ b/src/base/worker_thread_ttd_after_io_op.c @@ -212,7 +212,7 @@ xdd_e2e_after_io_op(worker_data_t *wdp) { if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_after_io_op: Target: %d: Worker: %d: ENTER\n", (long long int)pclk_now(),tdp->td_target_number,wdp->wd_worker_number); if (xgp->global_options & GO_DEBUG_E2E) xdd_show_task(&wdp->wd_task); - if ( (wdp->wd_task.task_io_status > 0) && (tdp->td_target_options & TO_ENDTOEND) ) { + if (wdp->wd_task.task_io_status > 0) { if (tdp->td_target_options & TO_E2E_SOURCE) { // For Serial Ordering, wait for the Previous I/O to complete before the associated Worker Thread releases this Worker Thread. // It is important to note that for Srial Ordering, when we get released by the Previous Worker Thread @@ -225,18 +225,20 @@ if (xgp->global_options & GO_DEBUG_E2E) xdd_show_task(&wdp->wd_task); wdp->wd_e2ep->e2e_hdrp->e2eh_magic = XDD_E2E_DATA_READY; wdp->wd_current_state |= WORKER_CURRENT_STATE_SRC_SEND; - if (PLAN_ENABLE_XNI & tdp->td_planp->plan_options) { - xint_e2e_xni_send(wdp); - } - else { -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_after_io_op: Target: %d: Worker: %d: Calling xdd_e2e_src_send...\n", (long long int)pclk_now(),tdp->td_target_number,wdp->wd_worker_number); - xdd_e2e_src_send(wdp); -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_after_io_op: Target: %d: Worker: %d: Returned from xdd_e2e_src_send...\n", (long long int)pclk_now(),tdp->td_target_number,wdp->wd_worker_number); - } - wdp->wd_current_state &= ~WORKER_CURRENT_STATE_SRC_SEND; + xint_e2e_xni_send(wdp); + wdp->wd_current_state &= ~WORKER_CURRENT_STATE_SRC_SEND; } // End of me being the SOURCE in an End-to-End test - } // End of processing a End-to-End + } + + if (tdp->td_target_options & TO_E2E_DESTINATION) { + // Release the current target buffer to XNI + xni_release_target_buffer(&wdp->wd_e2ep->xni_wd_buf); + wdp->wd_e2ep->e2e_hdrp = NULL; + wdp->wd_task.task_datap = NULL; + wdp->wd_e2ep->e2e_datap = NULL; + } + if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_after_io_op: Target: %d: Worker: %d: EXIT...\n", (long long int)pclk_now(),tdp->td_target_number,wdp->wd_worker_number); } // End of xdd_e2e_after_io_op(wdp) @@ -340,8 +342,10 @@ xdd_worker_thread_ttd_after_io_op(worker_data_t *wdp) { // Read-After_Write Processing xdd_raw_after_io_op(wdp); - // End-to-End Processing - xdd_e2e_after_io_op(wdp); + if (xint_is_e2e(tdp)) { + // End-to-End Processing + xdd_e2e_after_io_op(wdp); + } // Extended Statistics xdd_extended_stats(wdp); diff --git a/src/common/end_to_end.h b/src/common/end_to_end.h index 202cd2f9..a206b204 100644 --- a/src/common/end_to_end.h +++ b/src/common/end_to_end.h @@ -75,9 +75,6 @@ struct xint_td_e2e { time_t e2e_src_file_mtime; // stat -c %Y *e2e_src_file_path, i.e., last modification time in_addr_t e2e_dest_addr; // Destination Address number of the E2E socket in_port_t e2e_dest_port; // Port number to use for the E2E socket - - /* XNI data */ - xni_connection_t xni_conn; }; typedef struct xint_td_e2e xint_td_e2e_t; diff --git a/src/common/target_data.c b/src/common/target_data.c index 00a89b30..215b10e3 100644 --- a/src/common/target_data.c +++ b/src/common/target_data.c @@ -107,6 +107,7 @@ xdd_init_new_target_data(target_data_t *tdp, int32_t n) { tdp->io_buffers = NULL; tdp->io_buffers_count = 0; + tdp->io_buffer_size = 0; tdp->xni_ibdevice = DEFAULT_IB_DEVICE; /* can be changed by '-ibdevice' CLO */ diff --git a/src/common/xint_td.h b/src/common/xint_td.h index 3b321737..44353d27 100644 --- a/src/common/xint_td.h +++ b/src/common/xint_td.h @@ -169,7 +169,10 @@ struct xint_target_data { // I/O buffers allocated and freed by the target thread but shared // with either XNI or the workers unsigned char **io_buffers; + // number of I/O buffers size_t io_buffers_count; + // size of each I/O buffer in bytes + size_t io_buffer_size; /* XNI Networking components */ xni_protocol_t xni_pcl; diff --git a/src/xnet/xnet_end_to_end.c b/src/xnet/xnet_end_to_end.c index 6b80a1fe..b1035b6d 100644 --- a/src/xnet/xnet_end_to_end.c +++ b/src/xnet/xnet_end_to_end.c @@ -63,7 +63,16 @@ int32_t xint_e2e_src_connect(target_data_t *tdp) { /* Create an XNI endpoint from the e2e spec */ xni_endpoint_t xep = {.host = ip_string, .port = tdp->td_e2ep->e2e_address_table[e2e_idx].base_port}; - rc = xni_connect(tdp->xni_ctx, &xep, &tdp->td_e2ep->xni_td_conn); + + // initialize the set of I/O buffers + xni_bufset_t bufset; + memset(&bufset, 0, sizeof(bufset)); + bufset.bufs = tdp->io_buffers; + bufset.bufcount = tdp->io_buffers_count; + bufset.bufsize = tdp->io_buffer_size; + bufset.reserved = getpagesize(); + + rc = xni_connect(tdp->xni_ctx, &xep, &bufset, &tdp->td_e2ep->xni_td_conn); return rc; } @@ -92,7 +101,16 @@ int32_t xint_e2e_dest_connect(target_data_t *tdp) { /* Create an XNI endpoint from the e2e spec */ xni_endpoint_t xep = {.host = ip_string, .port = tdp->td_e2ep->e2e_address_table[e2e_idx].base_port}; - rc = xni_accept_connection(tdp->xni_ctx, &xep, &tdp->td_e2ep->xni_td_conn); + + // initialize the set of I/O buffers + xni_bufset_t bufset; + memset(&bufset, 0, sizeof(bufset)); + bufset.bufs = tdp->io_buffers; + bufset.bufcount = tdp->io_buffers_count; + bufset.bufsize = tdp->io_buffer_size; + bufset.reserved = getpagesize(); + + rc = xni_accept_connection(tdp->xni_ctx, &xep, &bufset, &tdp->td_e2ep->xni_td_conn); return rc; } @@ -152,14 +170,16 @@ int32_t xint_e2e_xni_send(worker_data_t *wdp) { /* Don't send magic eof across wire */ if (XDD_E2E_EOF != e2ehp->e2eh_magic) { /* Set XNI parameters and send */ - xni_target_buffer_set_data_length(e2ep->e2e_xfer_size, - e2ep->xni_wd_buf); + xni_target_buffer_set_sequence_number(e2ehp->e2eh_sequence_number, + e2ep->xni_wd_buf); xni_target_buffer_set_target_offset(e2ehp->e2eh_byte_offset, e2ep->xni_wd_buf); + xni_target_buffer_set_data_length(e2ep->e2e_xfer_size, + e2ep->xni_wd_buf); e2ep->e2e_send_status = xni_send_target_buffer(tdp->td_e2ep->xni_td_conn, &e2ep->xni_wd_buf); /* Request a fresh buffer from XNI */ - xni_request_target_buffer(tdp->xni_ctx, &wdp->wd_e2ep->xni_wd_buf); + xni_request_target_buffer(tdp->td_e2ep->xni_td_conn, &wdp->wd_e2ep->xni_wd_buf); /* The first page is XNI, the second page is E2E header */ uintptr_t bufp = @@ -215,9 +235,6 @@ int32_t xint_e2e_xni_recv(worker_data_t *wdp) { nclk_t e2e_wait_1st_msg_start_time; // This is the time stamp of when the first message arrived xdd_ts_tte_t *ttep; // Pointer to a time stamp table entry - /* Release the current target buffer to XNI */ - xni_release_target_buffer(&wdp->wd_e2ep->xni_wd_buf); - /* Collect the begin time */ nclk_now(&e2e_wait_1st_msg_start_time); wdp->wd_counters.tc_current_net_start_time = e2e_wait_1st_msg_start_time; @@ -289,7 +306,7 @@ int32_t xint_e2e_xni_recv(worker_data_t *wdp) { int xint_is_e2e(const target_data_t *tdp) { - return (1 == (TO_ENDTOEND & tdp->td_target_options)); + return (TO_ENDTOEND == (TO_ENDTOEND & tdp->td_target_options)); } /* diff --git a/src/xni/xni.c b/src/xni/xni.c index 291c60b6..f912313f 100644 --- a/src/xni/xni.c +++ b/src/xni/xni.c @@ -25,20 +25,15 @@ int xni_context_destroy(xni_context_t* ctx) return (*ctx)->protocol->context_destroy(ctx); } -int xni_accept_connection(xni_context_t ctx, struct xni_endpoint* local, xni_connection_t* conn) +int xni_accept_connection(xni_context_t ctx, struct xni_endpoint* local, xni_bufset_t *bufset, xni_connection_t* conn) { - return ctx->protocol->accept_connection(ctx, local, conn); -} - -int xni_register_buffer(xni_context_t ctx, void* buf, size_t nbytes, size_t reserved) -{ - return ctx->protocol->register_buffer(ctx, buf, nbytes, reserved); + return ctx->protocol->accept_connection(ctx, local, bufset, conn); } //TODO: local_endpoint -int xni_connect(xni_context_t ctx, struct xni_endpoint* remote, xni_connection_t* conn) +int xni_connect(xni_context_t ctx, struct xni_endpoint* remote, xni_bufset_t *bufset, xni_connection_t* conn) { - return ctx->protocol->connect(ctx, remote, conn); + return ctx->protocol->connect(ctx, remote, bufset, conn); } int xni_close_connection(xni_connection_t* conn) @@ -46,10 +41,10 @@ int xni_close_connection(xni_connection_t* conn) return (*conn)->context->protocol->close_connection(conn); } -int xni_request_target_buffer(xni_context_t context, +int xni_request_target_buffer(xni_connection_t conn, xni_target_buffer_t* buffer) { - return context->protocol->request_target_buffer(context, buffer); + return conn->context->protocol->request_target_buffer(conn, buffer); } int xni_send_target_buffer(xni_connection_t conn, xni_target_buffer_t* buffer) @@ -64,7 +59,7 @@ int xni_receive_target_buffer(xni_connection_t conn, xni_target_buffer_t* buffer int xni_release_target_buffer(xni_target_buffer_t* buffer) { - return (*buffer)->context->protocol->release_target_buffer(buffer); + return (*buffer)->connection->context->protocol->release_target_buffer(buffer); } void *xni_target_buffer_data(xni_target_buffer_t tb) diff --git a/src/xni/xni.h b/src/xni/xni.h index d47840e1..8f55b730 100644 --- a/src/xni/xni.h +++ b/src/xni/xni.h @@ -88,6 +88,19 @@ struct xni_target_buffer; */ typedef struct xni_target_buffer *xni_target_buffer_t; +struct xni_bufset { + /*! An array of buffer base addresses. */ + unsigned char **bufs; + /*! The number of elements in \c bufs */ + size_t bufcount; + /*! The size of each buffer in bytes. */ + size_t bufsize; + /*! The offset into the buffer at which the caller will insert + application data. */ + size_t reserved; +}; +/*! \brief Set of data buffers. */ +typedef struct xni_bufset xni_bufset_t; /*! \brief Perform library-specific initialization. * @@ -145,34 +158,15 @@ int xni_context_create(xni_protocol_t protocol, xni_control_block_t control_bloc */ int xni_context_destroy(xni_context_t *context); -/*! \brief Register memory with XNI. - * - * This function provides XNI drivers to perform optimizations based on - * the address of the memory buffers in use. - * - * \param context The context to register the buffer with. - * \param[in] buf The memory buffer to register. - * \param nbytes The total size of the buffer in bytes. - * \param reserved The offset into the buffer at which the caller will - * insert application data. Although this seems backwards, it ensures both - * the caller and XNI can align data per their own requirements. - * - * \return #XNI_OK if registration was successful. - * \return #XNI_ERR if registration failed. - */ -int xni_register_buffer(xni_context_t context, void* buf, size_t nbytes, size_t reserved); - /*! \brief Wait for a connection from a remote process. * * This function creates a destination-side connection by * listening on the address specified by \e local for a connection * from a remote process. * - * It is forbidden for the \e num_buffers and \e buffer_size arguments - * to differ from those specified at the remote end to xni_connect(). - * * \param context The network context under which to create the connection. * \param[in] local The local address to listen on. + * \param[in] bufset The buffers to be used for communication with the remote. * \param[out] connection The newly created destination-side connection. * * \return #XNI_OK if the connection was successfully created. @@ -181,7 +175,7 @@ int xni_register_buffer(xni_context_t context, void* buf, size_t nbytes, size_t * \sa xni_close_connection() * \sa xni_receive_target_buffer() */ -int xni_accept_connection(xni_context_t context, struct xni_endpoint *local, xni_connection_t *connection); +int xni_accept_connection(xni_context_t context, struct xni_endpoint *local, xni_bufset_t *bufset, xni_connection_t *connection); /*! \brief Initiate a connection to a remote process. * * This function creates a source-side connection by @@ -191,12 +185,9 @@ int xni_accept_connection(xni_context_t context, struct xni_endpoint *local, xni * context was created. These buffers will be aligned on 512-byte * boundaries. * - * It is forbidden for the \e num_buffers and \e buffer_size arguments - * to differ from those specified at the remote end to - * xni_accept_connection(). - * * \param context The network context under which to create the connection. * \param[in] remote The remote address to connect to. + * \param[in] bufset The buffers to be used for communication with the remote. * \param[out] connection The newly created source-side connection. * * \return #XNI_OK if the connection was successfully created. @@ -206,7 +197,7 @@ int xni_accept_connection(xni_context_t context, struct xni_endpoint *local, xni * \sa xni_request_target_buffer() */ //TODO: local_endpoint -int xni_connect(xni_context_t context, struct xni_endpoint *remote, xni_connection_t *connection); +int xni_connect(xni_context_t context, struct xni_endpoint *remote, xni_bufset_t *bufset, xni_connection_t *connection); /*! \brief Close a connection and free its resources. * * This function closes a connection and frees all allocated target @@ -237,8 +228,8 @@ int xni_close_connection(xni_connection_t *connection); * temporarily owned by the caller until the buffer is passed to * xni_send_target_buffer() or xni_release_target_buffer(). * - * \param contest The source-side context from which to request - * the buffer. + * \param connection The source-side connection from which to + * request the buffer. * \param[out] buffer The requested target buffer. * * \return #XNI_OK if a target buffer was reserved. @@ -246,7 +237,7 @@ int xni_close_connection(xni_connection_t *connection); * * \sa xni_send_target_buffer(); */ -int xni_request_target_buffer(xni_context_t ctx, xni_target_buffer_t *buffer); +int xni_request_target_buffer(xni_connection_t conn, xni_target_buffer_t *buffer); /*! \brief Send a target buffer to the remote process. * * This function transfers the target buffer \e buffer to the remote diff --git a/src/xni/xni_ib.c b/src/xni/xni_ib.c index 4cab8d74..58cb3683 100644 --- a/src/xni/xni_ib.c +++ b/src/xni/xni_ib.c @@ -38,17 +38,6 @@ struct ib_context { struct ib_control_block control_block; struct ibv_context *verbs_context; struct ibv_pd *domain; - - // target buffer registration data - struct ib_target_buffer *target_buffers; - size_t num_registered; - - // locks - pthread_mutex_t target_buffers_mutex; - pthread_cond_t target_buffers_cond; - pthread_mutex_t busy_flag_mutex; - pthread_cond_t busy_flag_cond; - }; struct ib_connection { @@ -74,6 +63,16 @@ struct ib_connection { uint32_t remote_qpnum; uint16_t remote_lid; + + // target buffer registration data + struct ib_target_buffer *target_buffers; + size_t num_registered; + + // locks + pthread_mutex_t target_buffers_mutex; + pthread_cond_t target_buffers_cond; + pthread_mutex_t busy_flag_mutex; + pthread_cond_t busy_flag_cond; }; #define IB_DATA_MESSAGE_HEADER_SIZE 20 // = tag(4) + @@ -86,7 +85,7 @@ enum send_state { }; struct ib_target_buffer { // inherited from xni_target_buffer - struct ib_context *context; + struct ib_connection *connection; void *data; int64_t sequence_number; size_t target_offset; @@ -98,7 +97,6 @@ struct ib_target_buffer { enum send_state send_state; struct ibv_mr *memory_region; void *header; - struct ib_connection *connection; }; #define IB_CREDIT_MESSAGE_SIZE 8 // = tag(4) + credits(4) @@ -183,12 +181,6 @@ static int ib_context_create(xni_protocol_t proto_, xni_control_block_t cb_, xni tmp->control_block = *cb; tmp->verbs_context = verbsctx; tmp->domain = pd; - tmp->target_buffers = calloc(cb->num_buffers, sizeof(*tmp->target_buffers)); - tmp->num_registered = 0; - pthread_mutex_init(&tmp->target_buffers_mutex, NULL); - pthread_cond_init(&tmp->target_buffers_cond, NULL); - pthread_mutex_init(&tmp->busy_flag_mutex, NULL); - pthread_cond_init(&tmp->busy_flag_cond, NULL); *ctx = tmp; return XNI_OK; @@ -207,41 +199,37 @@ static int ib_context_destroy(xni_context_t *ctx_) return XNI_OK; } -static int ib_register_buffer(xni_context_t ctx_, void* buf, size_t nbytes, size_t reserved) +static int register_buffer(struct ib_connection *conn, void* buf, size_t nbytes, size_t reserved) { - struct ib_context* ctx = (struct ib_context*)ctx_; uintptr_t beginp = (uintptr_t)buf; uintptr_t datap = (uintptr_t)buf + (uintptr_t)(reserved); size_t avail = (size_t)(datap - beginp); - // Make sure space exists in the registered buffer array - if (ctx->control_block.num_buffers <= ctx->num_registered) - return XNI_ERR; - // Make sure enough padding exists if (avail < IB_DATA_MESSAGE_HEADER_SIZE) return XNI_ERR; // Register the memory with verbs - struct ibv_mr *mr = ibv_reg_mr(ctx->domain, buf, nbytes, + struct ibv_mr *mr = ibv_reg_mr(conn->context->domain, buf, nbytes, IBV_ACCESS_LOCAL_WRITE); if (NULL == mr) return XNI_ERR; // Add the buffer into the array of registered buffers - pthread_mutex_lock(&ctx->target_buffers_mutex); - struct ib_target_buffer* tb = ctx->target_buffers + ctx->num_registered; - tb->context = ctx; + pthread_mutex_lock(&conn->target_buffers_mutex); + struct ib_target_buffer* tb = conn->target_buffers + conn->num_registered; + tb->connection = conn; tb->data = (void*)datap; + tb->sequence_number = 0; + tb->target_offset = 0; tb->data_length = -1; tb->buffer_size = nbytes - reserved; tb->busy = 0; - tb->send_state = 0; + tb->send_state = QUEUED; tb->memory_region = mr; tb->header = (void*)(datap - IB_DATA_MESSAGE_HEADER_SIZE); - tb->connection = NULL; - ctx->num_registered++; - pthread_mutex_unlock(&ctx->target_buffers_mutex); + conn->num_registered++; + pthread_mutex_unlock(&conn->target_buffers_mutex); return XNI_OK; } @@ -402,7 +390,7 @@ static int send_credits(struct ib_connection *conn, int ncredits) // otherwise mark any that have become free if (cb == NULL) { - size_t num_bufs = conn->context->num_registered; + size_t num_bufs = conn->num_registered; struct ibv_wc wc[num_bufs]; int completed = ibv_poll_cq(conn->send_cq, num_bufs, wc); if (completed < 0) { @@ -499,7 +487,7 @@ static int send_eof(struct ib_connection *conn) { //TODO: use a better buffer //XXX: for now, just hijack the first target bufffer - struct ib_target_buffer *tb = conn->context->target_buffers; + struct ib_target_buffer *tb = conn->target_buffers; // encode the message memcpy(tb->header, EOF_MESSAGE_TAG, TAG_LENGTH); @@ -539,7 +527,7 @@ static int send_eof(struct ib_connection *conn) return 0; } -static int ib_accept_connection(xni_context_t ctx_, struct xni_endpoint* local, xni_connection_t* conn_) +static int ib_accept_connection(xni_context_t ctx_, struct xni_endpoint* local, xni_bufset_t *bufset, xni_connection_t* conn_) { struct ib_context *ctx = (struct ib_context*)ctx_; struct ib_connection **conn = (struct ib_connection**)conn_; @@ -550,23 +538,23 @@ static int ib_accept_connection(xni_context_t ctx_, struct xni_endpoint* local, struct ibv_qp *qp = NULL; int server=-1, client=-1; - // Ensure a registered buffer exists - if (ctx->num_registered < 1) + // Ensure at least one buffer exists + if (bufset->bufcount < 1) return XNI_ERR; tmpconn = calloc(1, sizeof(*tmpconn)); tmpconn->context = ctx; - credit_buffers = allocate_credit_buffers(ctx, ctx->num_registered); + credit_buffers = allocate_credit_buffers(ctx, bufset->bufcount); if (credit_buffers == NULL) goto error_out; - if ((sendcq = ibv_create_cq(ctx->verbs_context, ctx->num_registered, NULL, NULL, 0)) == NULL) + if ((sendcq = ibv_create_cq(ctx->verbs_context, bufset->bufcount, NULL, NULL, 0)) == NULL) goto error_out; - if ((recvcq = ibv_create_cq(ctx->verbs_context, ctx->num_registered, NULL, NULL, 0)) == NULL) + if ((recvcq = ibv_create_cq(ctx->verbs_context, bufset->bufcount, NULL, NULL, 0)) == NULL) goto error_out; - if ((qp = create_queue_pair(ctx, sendcq, recvcq, ctx->num_registered)) == NULL) + if ((qp = create_queue_pair(ctx, sendcq, recvcq, bufset->bufcount)) == NULL) goto error_out; // start listening for a client @@ -646,24 +634,41 @@ static int ib_accept_connection(xni_context_t ctx_, struct xni_endpoint* local, if (move_qp_to_init(qp)) goto error_out; - for (size_t i =0; i < ctx->num_registered; i++) { - struct ib_target_buffer* tbp = ctx->target_buffers + i; - tbp->connection = tmpconn; - size_t bufsiz = tbp->buffer_size; + // Prepare for target buffer registration + tmpconn->target_buffers = calloc(bufset->bufcount, sizeof(*tmpconn->target_buffers)); + tmpconn->num_registered = 0; + pthread_mutex_init(&tmpconn->target_buffers_mutex, NULL); + pthread_cond_init(&tmpconn->target_buffers_cond, NULL); + pthread_mutex_init(&tmpconn->busy_flag_mutex, NULL); + pthread_cond_init(&tmpconn->busy_flag_cond, NULL); + + // Register the target buffers + for (size_t i =0; i < bufset->bufcount; i++) { + int rc = register_buffer(tmpconn, bufset->bufs[i], + bufset->bufsize, bufset->reserved); + if (rc != XNI_OK) { + goto error_out; + } + } + + // Post the receives + for (size_t i = 0; i < tmpconn->num_registered; i++) { + struct ib_target_buffer *tbp = tmpconn->target_buffers + i; + const size_t bufsiz = tbp->buffer_size; if (post_receive(qp, tbp->memory_region, tbp->header, (int)((char*)(tbp->data) - (char*)(tbp->header) + bufsiz), (uintptr_t)tbp)) goto error_out; } - if (move_qp_to_rtr(qp, remote_qpnum, remote_lid, ctx->num_registered) || - move_qp_to_rts(qp, ctx->num_registered)) + + if (move_qp_to_rtr(qp, remote_qpnum, remote_lid, tmpconn->num_registered) || + move_qp_to_rts(qp, tmpconn->num_registered)) goto error_out; #ifdef XNI_TRACE puts("Connected."); #endif // XNI_TRACE - tmpconn->context = ctx; tmpconn->credit_buffers = credit_buffers; tmpconn->eof = 0; pthread_mutex_init(&tmpconn->credit_mutex, NULL); @@ -675,7 +680,7 @@ static int ib_accept_connection(xni_context_t ctx_, struct xni_endpoint* local, tmpconn->remote_lid = remote_lid; // send the initial credits - if (send_credits(tmpconn, ctx->num_registered)) + if (send_credits(tmpconn, tmpconn->num_registered)) goto error_out; *conn = tmpconn; @@ -701,7 +706,7 @@ static int ib_accept_connection(xni_context_t ctx_, struct xni_endpoint* local, return XNI_ERR; } -static int ib_connect(xni_context_t ctx_, struct xni_endpoint* remote, xni_connection_t* conn_) +static int ib_connect(xni_context_t ctx_, struct xni_endpoint* remote, xni_bufset_t *bufset, xni_connection_t* conn_) { struct ib_context *ctx = (struct ib_context*)ctx_; struct ib_connection **conn = (struct ib_connection**)conn_; @@ -712,8 +717,8 @@ static int ib_connect(xni_context_t ctx_, struct xni_endpoint* remote, xni_conne struct ibv_qp *qp = NULL; int server=-1, client=-1; - // Ensure a registered buffer exists - if (ctx->num_registered < 1) + // Ensure at least one buffer exists + if (bufset->bufcount < 1) return XNI_ERR; #ifdef XNI_TRACE @@ -727,19 +732,19 @@ static int ib_connect(xni_context_t ctx_, struct xni_endpoint* remote, xni_conne #ifdef XNI_TRACE puts("3"); #endif // XNI_TRACE - credit_buffers = allocate_credit_buffers(ctx, ctx->num_registered); + credit_buffers = allocate_credit_buffers(ctx, bufset->bufcount); if (credit_buffers == NULL) goto error_out; #ifdef XNI_TRACE puts("4"); #endif // XNI_TRACE - if ((sendcq = ibv_create_cq(ctx->verbs_context, ctx->num_registered, NULL, NULL, 0)) == NULL) + if ((sendcq = ibv_create_cq(ctx->verbs_context, bufset->bufcount, NULL, NULL, 0)) == NULL) goto error_out; - if ((recvcq = ibv_create_cq(ctx->verbs_context, ctx->num_registered, NULL, NULL, 0)) == NULL) + if ((recvcq = ibv_create_cq(ctx->verbs_context, bufset->bufcount, NULL, NULL, 0)) == NULL) goto error_out; - if ((qp = create_queue_pair(ctx, sendcq, recvcq, ctx->num_registered)) == NULL) + if ((qp = create_queue_pair(ctx, sendcq, recvcq, bufset->bufcount)) == NULL) goto error_out; #ifdef XNI_TRACE puts("Create qps"); @@ -810,8 +815,8 @@ static int ib_connect(xni_context_t ctx_, struct xni_endpoint* remote, xni_conne IB_CREDIT_MESSAGE_SIZE, (uintptr_t)*cbptr)) goto error_out; - if (move_qp_to_rtr(qp, remote_qpnum, remote_lid, ctx->num_registered) || - move_qp_to_rts(qp, ctx->num_registered)) + if (move_qp_to_rtr(qp, remote_qpnum, remote_lid, bufset->bufcount) || + move_qp_to_rts(qp, bufset->bufcount)) goto error_out; tmpconn->context = ctx; @@ -826,9 +831,22 @@ static int ib_connect(xni_context_t ctx_, struct xni_endpoint* remote, xni_conne tmpconn->remote_qpnum = remote_qpnum; tmpconn->remote_lid = remote_lid; - // Add the connection to the registered buffers - for (size_t i = 0; i < ctx->num_registered; i++) - ctx->target_buffers[i].connection = tmpconn; + // Prepare for target buffer registration + tmpconn->target_buffers = calloc(bufset->bufcount, sizeof(*tmpconn->target_buffers)); + tmpconn->num_registered = 0; + pthread_mutex_init(&tmpconn->target_buffers_mutex, NULL); + pthread_cond_init(&tmpconn->target_buffers_cond, NULL); + pthread_mutex_init(&tmpconn->busy_flag_mutex, NULL); + pthread_cond_init(&tmpconn->busy_flag_cond, NULL); + + // Register the target buffers + for (size_t i =0; i < bufset->bufcount; i++) { + int rc = register_buffer(tmpconn, bufset->bufs[i], + bufset->bufsize, bufset->reserved); + if (rc != XNI_OK) { + goto error_out; + } + } #ifdef XNI_TRACE puts("Connected."); @@ -880,16 +898,16 @@ static int ib_close_connection(xni_connection_t *conn_) return XNI_OK; } -static int ib_request_target_buffer(xni_context_t ctx_, xni_target_buffer_t *targetbuf_) +static int ib_request_target_buffer(xni_connection_t conn_, xni_target_buffer_t *targetbuf_) { - struct ib_context *ctx = (struct ib_context*)ctx_; + struct ib_connection *conn = (struct ib_connection*)conn_; struct ib_target_buffer **targetbuf = (struct ib_target_buffer**)targetbuf_; struct ib_target_buffer *tb = NULL; - pthread_mutex_lock(&ctx->busy_flag_mutex); + pthread_mutex_lock(&conn->busy_flag_mutex); while (tb == NULL) { - for (size_t i = 0; i < ctx->num_registered; i++) { - struct ib_target_buffer* ptr = ctx->target_buffers + i; + for (size_t i = 0; i < conn->num_registered; i++) { + struct ib_target_buffer* ptr = conn->target_buffers + i; if (!ptr->busy) { tb = ptr; @@ -898,10 +916,10 @@ static int ib_request_target_buffer(xni_context_t ctx_, xni_target_buffer_t *tar } } if (tb == NULL) - pthread_cond_wait(&ctx->busy_flag_cond, - &ctx->busy_flag_mutex); + pthread_cond_wait(&conn->busy_flag_cond, + &conn->busy_flag_mutex); } - pthread_mutex_unlock(&ctx->busy_flag_mutex); + pthread_mutex_unlock(&conn->busy_flag_mutex); *targetbuf = tb; return XNI_OK; @@ -951,7 +969,7 @@ static int ib_send_target_buffer(xni_connection_t conn_, xni_target_buffer_t *ta // wait for send completion pthread_mutex_lock(&conn->send_state_mutex); while (tb->send_state == QUEUED) { - size_t num_bufs = conn->context->num_registered; + size_t num_bufs = conn->num_registered; struct ibv_wc wc[num_bufs]; int completed = ibv_poll_cq(conn->send_cq, num_bufs, wc); if (completed < 0) { @@ -975,10 +993,10 @@ static int ib_send_target_buffer(xni_connection_t conn_, xni_target_buffer_t *ta free_out: // mark the buffer as free - pthread_mutex_lock(&conn->context->busy_flag_mutex); + pthread_mutex_lock(&conn->busy_flag_mutex); tb->busy = 0; - pthread_cond_signal(&conn->context->busy_flag_cond); - pthread_mutex_unlock(&conn->context->busy_flag_mutex); + pthread_cond_signal(&conn->busy_flag_cond); + pthread_mutex_unlock(&conn->busy_flag_mutex); return return_code; } @@ -1045,10 +1063,10 @@ static int ib_release_target_buffer(xni_target_buffer_t *targetbuf_) if (send_credits(tb->connection, 1)) return XNI_ERR; } else { - pthread_mutex_lock(&tb->connection->context->busy_flag_mutex); + pthread_mutex_lock(&tb->connection->busy_flag_mutex); tb->busy = 0; - pthread_cond_signal(&tb->connection->context->busy_flag_cond); - pthread_mutex_unlock(&tb->connection->context->busy_flag_mutex); + pthread_cond_signal(&tb->connection->busy_flag_cond); + pthread_mutex_unlock(&tb->connection->busy_flag_mutex); } *targetbuf = NULL; @@ -1060,7 +1078,6 @@ static struct xni_protocol protocol_ib = { .name = PROTOCOL_NAME, .context_create = ib_context_create, .context_destroy = ib_context_destroy, - .register_buffer = ib_register_buffer, .accept_connection = ib_accept_connection, .connect = ib_connect, .close_connection = ib_close_connection, diff --git a/src/xni/xni_internal.h b/src/xni/xni_internal.h index 8be393ef..f2979365 100644 --- a/src/xni/xni_internal.h +++ b/src/xni/xni_internal.h @@ -10,13 +10,11 @@ struct xni_protocol { int (*context_create)(xni_protocol_t, xni_control_block_t, xni_context_t*); int (*context_destroy)(xni_context_t*); - int (*register_buffer)(xni_context_t, void*, size_t, size_t); - - int (*accept_connection)(xni_context_t, struct xni_endpoint*, xni_connection_t*); - int (*connect)(xni_context_t, struct xni_endpoint*, xni_connection_t*); + int (*accept_connection)(xni_context_t, struct xni_endpoint*, xni_bufset_t*, xni_connection_t*); + int (*connect)(xni_context_t, struct xni_endpoint*, xni_bufset_t*, xni_connection_t*); int (*close_connection)(xni_connection_t*); - int (*request_target_buffer)(xni_context_t, xni_target_buffer_t*); + int (*request_target_buffer)(xni_connection_t, xni_target_buffer_t*); int (*send_target_buffer)(xni_connection_t, xni_target_buffer_t*); int (*receive_target_buffer)(xni_connection_t, xni_target_buffer_t*); int (*release_target_buffer)(xni_target_buffer_t*); @@ -31,7 +29,7 @@ struct xni_connection { }; struct xni_target_buffer { - struct xni_context *context; + struct xni_connection *connection; void *data; int64_t sequence_number; size_t target_offset; diff --git a/src/xni/xni_tcp.c b/src/xni/xni_tcp.c index 1ddfb5eb..4a708dd4 100644 --- a/src/xni/xni_tcp.c +++ b/src/xni/xni_tcp.c @@ -41,13 +41,6 @@ struct tcp_context { // added by struct tcp_context struct tcp_control_block control_block; - - // added by struct tcp_connection - struct tcp_target_buffer *registered_buffers; // NULL-terminated - size_t num_registered; - pthread_mutex_t buffer_mutex; - pthread_cond_t buffer_cond; - }; struct tcp_socket { @@ -60,17 +53,27 @@ struct tcp_connection { // inherited from struct xni_connection struct tcp_context *context; + // added by struct tcp_connection + // 1 = destination side, 0 = source side int destination; struct tcp_socket *sockets; int num_sockets; pthread_mutex_t socket_mutex; pthread_cond_t socket_cond; + + // size is equal to num_sockets + struct tcp_target_buffer *registered_buffers; + // count of registered buffers + size_t num_registered; + // these protect registered_buffers and num_registered + pthread_mutex_t buffer_mutex; + pthread_cond_t buffer_cond; }; struct tcp_target_buffer { // inherited from xni_target_buffer - struct tcp_context *context; + struct tcp_connection *connection; void *data; int64_t sequence_number; size_t target_offset; @@ -109,28 +112,22 @@ int xni_allocate_tcp_control_block(int num_sockets, const char *congestion, int int xni_free_tcp_control_block(xni_control_block_t *cb_) { struct tcp_control_block **cb = (struct tcp_control_block**)cb_; - free(*cb); *cb = NULL; return XNI_OK; } -static int tcp_context_create(xni_protocol_t proto_, xni_control_block_t cb_, xni_context_t *ctx_) +static int tcp_context_create(xni_protocol_t proto, xni_control_block_t cb_, xni_context_t *ctx_) { - struct xni_protocol *proto = proto_; struct tcp_control_block *cb = (struct tcp_control_block*)cb_; struct tcp_context **ctx = (struct tcp_context**)ctx_; - size_t num_buffers = cb->num_sockets; assert(strcmp(proto->name, PROTOCOL_NAME) == 0); // fill in a new context struct tcp_context *tmp = calloc(1, sizeof(*tmp)); tmp->protocol = proto; tmp->control_block = *cb; - tmp->registered_buffers = calloc(num_buffers, sizeof(*tmp->registered_buffers)); - pthread_mutex_init(&tmp->buffer_mutex, NULL); - pthread_cond_init(&tmp->buffer_cond, NULL); // Swap in the context *ctx = tmp; @@ -141,23 +138,23 @@ static int tcp_context_create(xni_protocol_t proto_, xni_control_block_t cb_, xn static int tcp_context_destroy(xni_context_t *ctx_) { struct tcp_context **ctx = (struct tcp_context **)ctx_; - pthread_mutex_destroy(&(*ctx)->buffer_mutex); - pthread_cond_destroy(&(*ctx)->buffer_cond); - free((*ctx)->registered_buffers); free(*ctx); *ctx = NULL; return XNI_OK; } -static int tcp_register_buffer(xni_context_t ctx_, void* buf, size_t nbytes, size_t reserved) { - struct tcp_context* ctx = (struct tcp_context*) ctx_; +static int register_buffer(struct tcp_connection *conn, void* buf, size_t nbytes, size_t reserved) { + // buffer base address uintptr_t beginp = (uintptr_t)buf; + // buffer data area base address uintptr_t datap = (uintptr_t)buf + (uintptr_t)(reserved); + // number of bytes available for headers + //TODO: isn't this just the same as `reserved' ? size_t avail = (size_t)(datap - beginp); // Make sure space exists in the registered buffers array - if (ctx->control_block.num_sockets <= ctx->num_registered) + if (conn->context->control_block.num_sockets <= conn->num_registered) return XNI_ERR; // Make sure enough padding exists @@ -165,22 +162,22 @@ static int tcp_register_buffer(xni_context_t ctx_, void* buf, size_t nbytes, siz return XNI_ERR; // Add the buffer into the array of registered buffers - pthread_mutex_lock(&ctx->buffer_mutex); - struct tcp_target_buffer *tb = ctx->registered_buffers + ctx->num_registered; - tb->context = ctx; + pthread_mutex_lock(&conn->buffer_mutex); + struct tcp_target_buffer *tb = conn->registered_buffers + conn->num_registered; + tb->connection = conn; tb->data = (void*)datap; + tb->sequence_number = 0; tb->target_offset = 0; tb->data_length = -1; tb->busy = 0; tb->header = (void*)(datap - TCP_DATA_MESSAGE_HEADER_SIZE); - ctx->registered_buffers[ctx->num_registered] = *tb; - ctx->num_registered++; - pthread_mutex_unlock(&ctx->buffer_mutex); + conn->num_registered++; + pthread_mutex_unlock(&conn->buffer_mutex); return XNI_OK; } -static int tcp_accept_connection(xni_context_t ctx_, struct xni_endpoint* local, xni_connection_t* conn_) +static int tcp_accept_connection(xni_context_t ctx_, struct xni_endpoint* local, xni_bufset_t *bufset, xni_connection_t* conn_) { struct tcp_context *ctx = (struct tcp_context*)ctx_; struct tcp_connection **conn = (struct tcp_connection**)conn_; @@ -284,6 +281,18 @@ static int tcp_accept_connection(xni_context_t ctx_, struct xni_endpoint* local, tmpconn->num_sockets = num_sockets; pthread_mutex_init(&tmpconn->socket_mutex, NULL); pthread_cond_init(&tmpconn->socket_cond, NULL); + tmpconn->registered_buffers = calloc(ctx->control_block.num_sockets, + sizeof(*tmpconn->registered_buffers)); + tmpconn->num_registered = 0; + pthread_mutex_init(&tmpconn->buffer_mutex, NULL); + pthread_cond_init(&tmpconn->buffer_cond, NULL); + + for (size_t i = 0; i < bufset->bufcount; i++) { + int rc = register_buffer(tmpconn, bufset->bufs[i], bufset->bufsize, bufset->reserved); + if (rc != XNI_OK) { + goto error_out; + } + } *conn = tmpconn; return XNI_OK; @@ -299,18 +308,17 @@ static int tcp_accept_connection(xni_context_t ctx_, struct xni_endpoint* local, if (servers[i] != -1) close(servers[i]); - /*// free any allocated target buffers - for (struct tcp_target_buffer **ptr = target_buffers; *ptr; ptr++) - ctx->control_block.free_fn(*ptr); + if (tmpconn) { + free(tmpconn->registered_buffers); + free(tmpconn); + } - free(target_buffers);*/ - free(tmpconn); free(clients); return XNI_ERR; } -static int tcp_connect(xni_context_t ctx_, struct xni_endpoint* remote, xni_connection_t* conn_) +static int tcp_connect(xni_context_t ctx_, struct xni_endpoint* remote, xni_bufset_t *bufset, xni_connection_t* conn_) { struct tcp_context *ctx = (struct tcp_context*)ctx_; struct tcp_connection **conn = (struct tcp_connection**)conn_; @@ -384,14 +392,24 @@ static int tcp_connect(xni_context_t ctx_, struct xni_endpoint* remote, xni_conn } } - //TODO: allocate target buffer list and attach - tmpconn->context = ctx; tmpconn->destination = 0; tmpconn->sockets = servers; tmpconn->num_sockets = num_sockets; pthread_mutex_init(&tmpconn->socket_mutex, NULL); pthread_cond_init(&tmpconn->socket_cond, NULL); + tmpconn->registered_buffers = calloc(ctx->control_block.num_sockets, + sizeof(*tmpconn->registered_buffers)); + tmpconn->num_registered = 0; + pthread_mutex_init(&tmpconn->buffer_mutex, NULL); + pthread_cond_init(&tmpconn->buffer_cond, NULL); + + for (size_t i = 0; i < bufset->bufcount; i++) { + int rc = register_buffer(tmpconn, bufset->bufs[i], bufset->bufsize, bufset->reserved); + if (rc != XNI_OK) { + goto error_out; + } + } *conn = tmpconn; return XNI_OK; @@ -402,12 +420,11 @@ static int tcp_connect(xni_context_t ctx_, struct xni_endpoint* remote, xni_conn if (servers[i].sockd != -1) close(servers[i].sockd); - // free any allocated target buffers - //for (struct tcp_target_buffer **ptr = target_buffers; *ptr; ptr++) - // ctx->control_block.free_fn(*ptr); + if (tmpconn) { + free(tmpconn->registered_buffers); + free(tmpconn); + } - //free(target_buffers); - free(tmpconn); free(servers); return XNI_ERR; @@ -426,27 +443,26 @@ static int tcp_close_connection(xni_connection_t *conn_) if (c->sockets[i].sockd != -1) close(c->sockets[i].sockd); - /*BWS - struct tcp_target_buffer **buffers = (c->destination ? c->receive_buffers : c->send_buffers); - while (*buffers++) - c->context->control_block.free_fn(*buffers); - */ + pthread_mutex_destroy(&c->buffer_mutex); + pthread_cond_destroy(&c->buffer_cond); + free(c->registered_buffers); + free(c); *conn = NULL; return XNI_OK; } -static int tcp_request_target_buffer(xni_context_t ctx_, xni_target_buffer_t *targetbuf_) +static int tcp_request_target_buffer(xni_connection_t conn_, xni_target_buffer_t *targetbuf_) { - struct tcp_context *ctx = (struct tcp_context*)ctx_; + struct tcp_connection *conn = (struct tcp_connection*)conn_; struct tcp_target_buffer **targetbuf = (struct tcp_target_buffer**)targetbuf_; struct tcp_target_buffer *tb = NULL; - pthread_mutex_lock(&ctx->buffer_mutex); + pthread_mutex_lock(&conn->buffer_mutex); while (tb == NULL) { - for (size_t i = 0; i < ctx->num_registered; i++) { - struct tcp_target_buffer *ptr = ctx->registered_buffers + i; + for (size_t i = 0; i < conn->num_registered; i++) { + struct tcp_target_buffer *ptr = conn->registered_buffers + i; if (!ptr->busy) { tb = ptr; tb->busy = 1; @@ -454,9 +470,9 @@ static int tcp_request_target_buffer(xni_context_t ctx_, xni_target_buffer_t *ta } } if (tb == NULL) - pthread_cond_wait(&ctx->buffer_cond, &ctx->buffer_mutex); + pthread_cond_wait(&conn->buffer_cond, &conn->buffer_mutex); } - pthread_mutex_unlock(&ctx->buffer_mutex); + pthread_mutex_unlock(&conn->buffer_mutex); *targetbuf = tb; return XNI_OK; @@ -515,10 +531,10 @@ static int tcp_send_target_buffer(xni_connection_t conn_, xni_target_buffer_t *t pthread_mutex_unlock(&conn->socket_mutex); // mark the buffer as free - pthread_mutex_lock(&tb->context->buffer_mutex); + pthread_mutex_lock(&conn->buffer_mutex); tb->busy = 0; - pthread_cond_signal(&tb->context->buffer_cond); - pthread_mutex_unlock(&tb->context->buffer_mutex); + pthread_cond_signal(&conn->buffer_cond); + pthread_mutex_unlock(&conn->buffer_mutex); *targetbuf = NULL; return XNI_OK; @@ -532,10 +548,10 @@ static int tcp_receive_target_buffer(xni_connection_t conn_, xni_target_buffer_t // grab a free buffer struct tcp_target_buffer *tb = NULL; - pthread_mutex_lock(&conn->context->buffer_mutex); + pthread_mutex_lock(&conn->buffer_mutex); while (tb == NULL) { - for (size_t i = 0; i < conn->context->num_registered; i++) { - struct tcp_target_buffer *ptr = conn->context->registered_buffers + i; + for (size_t i = 0; i < conn->num_registered; i++) { + struct tcp_target_buffer *ptr = conn->registered_buffers + i; if (!ptr->busy) { tb = ptr; tb->busy = 1; @@ -543,9 +559,9 @@ static int tcp_receive_target_buffer(xni_connection_t conn_, xni_target_buffer_t } } if (tb == NULL) - pthread_cond_wait(&conn->context->buffer_cond, &conn->context->buffer_mutex); + pthread_cond_wait(&conn->buffer_cond, &conn->buffer_mutex); } - pthread_mutex_unlock(&conn->context->buffer_mutex); + pthread_mutex_unlock(&conn->buffer_mutex); struct tcp_socket *socket = NULL; while (socket == NULL) { @@ -647,10 +663,10 @@ static int tcp_receive_target_buffer(xni_connection_t conn_, xni_target_buffer_t buffer_out: if (return_code != XNI_OK) { // mark the buffer as free - pthread_mutex_lock(&conn->context->buffer_mutex); + pthread_mutex_lock(&conn->buffer_mutex); tb->busy = 0; - pthread_cond_signal(&conn->context->buffer_cond); - pthread_mutex_unlock(&conn->context->buffer_mutex); + pthread_cond_signal(&conn->buffer_cond); + pthread_mutex_unlock(&conn->buffer_mutex); } return return_code; @@ -664,10 +680,12 @@ static int tcp_release_target_buffer(xni_target_buffer_t *targetbuf_) tb->target_offset = 0; tb->data_length = -1; - pthread_mutex_lock(&tb->context->buffer_mutex); + //TODO: is it really OK to access the lock protecting an object + // through the object itself? + pthread_mutex_lock(&tb->connection->buffer_mutex); tb->busy = 0; - pthread_cond_signal(&tb->context->buffer_cond); - pthread_mutex_unlock(&tb->context->buffer_mutex); + pthread_cond_signal(&tb->connection->buffer_cond); + pthread_mutex_unlock(&tb->connection->buffer_mutex); *targetbuf = NULL; return XNI_OK; @@ -678,7 +696,6 @@ static struct xni_protocol protocol_tcp = { .name = PROTOCOL_NAME, .context_create = tcp_context_create, .context_destroy = tcp_context_destroy, - .register_buffer = tcp_register_buffer, .accept_connection = tcp_accept_connection, .connect = tcp_connect, .close_connection = tcp_close_connection, From aa5192c858b8934c7bf0bae8b06874fcf0466c74 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Thu, 10 Jul 2014 14:16:43 -0400 Subject: [PATCH 25/50] Assume buffer count is always tdp->td_queue_depth --- src/base/target_init.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/base/target_init.c b/src/base/target_init.c index 4f2a21b3..fdcfa907 100644 --- a/src/base/target_init.c +++ b/src/base/target_init.c @@ -20,12 +20,6 @@ xint_get_buffer_count(const target_data_t *tdp) { size_t count = tdp->td_queue_depth; - if (xint_is_e2e(tdp) && tdp->td_e2ep->e2e_address_table_port_count > 0) { - - count = tdp->td_e2ep->e2e_address_table_port_count; - } - - //TODO: is the buffer count always just tdp->td_queue_depth? return count; } From 76e6c79f902a9a715a9e269461acf199b2a23445 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Fri, 11 Jul 2014 15:37:37 -0400 Subject: [PATCH 26/50] Remove the E2E header Uses XNI to store and retrieve the information that was formerly held in the E2E header. The old end-to-end code is thoroughly broken now. XNI should be used in its place via the "-xni" option. --- src/base/io_buffers.c | 11 +- src/base/worker_thread_init.c | 7 +- src/base/worker_thread_io.c | 2 +- src/base/worker_thread_ttd_after_io_op.c | 2 - src/base/worker_thread_ttd_before_io_op.c | 23 +- src/client/interactive_func.c | 8 +- src/common/debug.c | 17 - src/common/end_to_end.h | 13 +- src/common/xint_prototypes.h | 8 - src/net/end_to_end.c | 599 +--------------------- src/net/end_to_end_init.c | 11 +- src/xnet/xnet_end_to_end.c | 88 ++-- 12 files changed, 55 insertions(+), 734 deletions(-) diff --git a/src/base/io_buffers.c b/src/base/io_buffers.c index 287a52d8..f018531a 100644 --- a/src/base/io_buffers.c +++ b/src/base/io_buffers.c @@ -60,15 +60,8 @@ xdd_init_io_buffers(target_data_t *tdp) { if (tdp->td_xfer_size % page_size) pages++; // Round up to page size if ((tdp->td_target_options & TO_ENDTOEND)) { - // Add one page for the e2e header - pages++; - - // If its XNI, add another page for XNI, better would be for XNI to - // pack all of the header data (and do the hton, ntoh calls) - xdd_plan_t *planp = tdp->td_planp; - if (PLAN_ENABLE_XNI & planp->plan_options) { - pages++; - } + // Add one page for the XNI header + pages++; } diff --git a/src/base/worker_thread_init.c b/src/base/worker_thread_init.c index afa5b2ba..aeafaf40 100644 --- a/src/base/worker_thread_init.c +++ b/src/base/worker_thread_init.c @@ -110,7 +110,6 @@ xdd_worker_thread_init(worker_data_t *wdp) { wdp->wd_e2ep->xni_wd_buf = NULL; wdp->wd_task.task_datap = NULL; wdp->wd_e2ep->e2e_datap = NULL; - wdp->wd_e2ep->e2e_hdrp = NULL; } else { //TODO: I'd really like to move all of these buffer // request calls to something like ttd_before_io_op, but I @@ -120,12 +119,8 @@ xdd_worker_thread_init(worker_data_t *wdp) { // Request an I/O buffer from XNI xni_request_target_buffer(tdp->td_e2ep->xni_td_conn, &wdp->wd_e2ep->xni_wd_buf); - unsigned char *bufp = xni_target_buffer_data(wdp->wd_e2ep->xni_wd_buf); - memset(bufp, 0, 2*getpagesize()); - // Use the first page for the E2E header - wdp->wd_task.task_datap = bufp + getpagesize(); + wdp->wd_task.task_datap = xni_target_buffer_data(wdp->wd_e2ep->xni_wd_buf); wdp->wd_e2ep->e2e_datap = wdp->wd_task.task_datap; - wdp->wd_e2ep->e2e_hdrp = (xdd_e2e_header_t *)(bufp + (getpagesize() - sizeof(xdd_e2e_header_t))); } } else { // For non-E2E operations the data portion is the entire buffer diff --git a/src/base/worker_thread_io.c b/src/base/worker_thread_io.c index 8eb24672..947ef6c7 100644 --- a/src/base/worker_thread_io.c +++ b/src/base/worker_thread_io.c @@ -57,7 +57,7 @@ xdd_worker_thread_io(worker_data_t *wdp) { // done receiving data from the Source Side. // If there is Loose or Serial Ordering in effect then we need to release the Next Worker Thread // before returning. - if ((tdp->td_target_options & TO_E2E_DESTINATION) && (wdp->wd_e2ep->e2e_hdrp->e2eh_magic == XDD_E2E_EOF)) { + if ((tdp->td_target_options & TO_E2E_DESTINATION) && wdp->wd_e2ep->received_eof) { // Indicate that this Worker Thread has received its EOF Packet pthread_mutex_lock(&wdp->wd_worker_thread_target_sync_mutex); wdp->wd_worker_thread_target_sync |= WTSYNC_EOF_RECEIVED; diff --git a/src/base/worker_thread_ttd_after_io_op.c b/src/base/worker_thread_ttd_after_io_op.c index d7ca05ff..898a8c39 100644 --- a/src/base/worker_thread_ttd_after_io_op.c +++ b/src/base/worker_thread_ttd_after_io_op.c @@ -222,7 +222,6 @@ if (xgp->global_options & GO_DEBUG_E2E) xdd_show_task(&wdp->wd_task); // operation. // Send the data to the Destination machine - wdp->wd_e2ep->e2e_hdrp->e2eh_magic = XDD_E2E_DATA_READY; wdp->wd_current_state |= WORKER_CURRENT_STATE_SRC_SEND; xint_e2e_xni_send(wdp); @@ -234,7 +233,6 @@ if (xgp->global_options & GO_DEBUG_E2E) xdd_show_task(&wdp->wd_task); if (tdp->td_target_options & TO_E2E_DESTINATION) { // Release the current target buffer to XNI xni_release_target_buffer(&wdp->wd_e2ep->xni_wd_buf); - wdp->wd_e2ep->e2e_hdrp = NULL; wdp->wd_task.task_datap = NULL; wdp->wd_e2ep->e2e_datap = NULL; } diff --git a/src/base/worker_thread_ttd_before_io_op.c b/src/base/worker_thread_ttd_before_io_op.c index 527a7a74..a09795bf 100644 --- a/src/base/worker_thread_ttd_before_io_op.c +++ b/src/base/worker_thread_ttd_before_io_op.c @@ -204,20 +204,11 @@ xdd_e2e_before_io_op(worker_data_t *wdp) { wdp->wd_e2ep->e2e_data_recvd = 0; // This will record how much data is recvd in this routine // Lets read a packet of data from the Source side - // The call to xdd_e2e_dest_recv() will block until there is data to read + // The call to xint_e2e_xni_recv() will block until there is data to read wdp->wd_current_state |= WORKER_CURRENT_STATE_DEST_RECEIVE; - if (PLAN_ENABLE_XNI & tdp->td_planp->plan_options) { - status = xint_e2e_xni_recv(wdp); - } - else { + status = xint_e2e_xni_recv(wdp); -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_before_io_op: Target: %d: Worker: %d: Calling xdd_e2e_dest_recv...\n ", (long long int)pclk_now(),tdp->td_target_number,wdp->wd_worker_number); - - status = xdd_e2e_dest_receive(wdp); - -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_before_io_op: Target: %d: Worker: %d: Returning from xdd_e2e_dest_recv: e2e header:\n ", (long long int)pclk_now(),tdp->td_target_number,wdp->wd_worker_number); - } wdp->wd_current_state &= ~WORKER_CURRENT_STATE_DEST_RECEIVE; // If status is "-1" then soemthing happened to the connection - time to leave @@ -225,17 +216,17 @@ if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e return(-1); // Check to see of this is the last message in the transmission - if (wdp->wd_e2ep->e2e_hdrp->e2eh_magic == XDD_E2E_EOF) { // This must be the End of the File + if (wdp->wd_e2ep->received_eof) { // This must be the End of the File return(0); } // Use the hearder.location as the new tdp->td_counters.tc_current_byte_offset and the e2e_header.length as the new my_current_xfer_size for this op // This will allow for the use of "no ordering" on the source side of an e2e operation - wdp->wd_task.task_byte_offset = wdp->wd_e2ep->e2e_hdrp->e2eh_byte_offset; - wdp->wd_task.task_xfer_size = wdp->wd_e2ep->e2e_hdrp->e2eh_data_length; - wdp->wd_task.task_op_number = wdp->wd_e2ep->e2e_hdrp->e2eh_sequence_number; + wdp->wd_task.task_byte_offset = xni_target_buffer_target_offset(wdp->wd_e2ep->xni_wd_buf); + wdp->wd_task.task_xfer_size = xni_target_buffer_data_length(wdp->wd_e2ep->xni_wd_buf); + wdp->wd_task.task_op_number = xni_target_buffer_sequence_number(wdp->wd_e2ep->xni_wd_buf); // Record the amount of data received - wdp->wd_e2ep->e2e_data_recvd = wdp->wd_e2ep->e2e_hdrp->e2eh_data_length; + wdp->wd_e2ep->e2e_data_recvd = xni_target_buffer_data_length(wdp->wd_e2ep->xni_wd_buf); return(0); diff --git a/src/client/interactive_func.c b/src/client/interactive_func.c index 0ba2bc63..2987bddd 100644 --- a/src/client/interactive_func.c +++ b/src/client/interactive_func.c @@ -323,15 +323,15 @@ xdd_interactive_display_state_info(worker_data_t *wdp) { if (wdp->wd_current_state & WORKER_CURRENT_STATE_DEST_RECEIVE) { fprintf(xgp->output," Destination Side of an E2E - waiting to receive data from Source, target op number %lld, location %lld, length %lld, recvfrom status is %d\n", (long long int)wdp->wd_counters.tc_current_op_number, - (long long int)wdp->wd_e2ep->e2e_hdrp->e2eh_byte_offset, - (long long int)wdp->wd_e2ep->e2e_hdrp->e2eh_data_length, + (long long int)xni_target_buffer_target_offset(wdp->wd_e2ep->xni_wd_buf), + (long long int)xni_target_buffer_data_length(wdp->wd_e2ep->xni_wd_buf), wdp->wd_e2ep->e2e_recv_status); } if (wdp->wd_current_state & WORKER_CURRENT_STATE_SRC_SEND) { fprintf(xgp->output," Source Side of an E2E - waiting to send data to Destination, target op number %lld, location %lld, length %lld, sendto status is %d\n", (long long int)wdp->wd_counters.tc_current_op_number, - (long long int)wdp->wd_e2ep->e2e_hdrp->e2eh_byte_offset, - (long long int)wdp->wd_e2ep->e2e_hdrp->e2eh_data_length, + (long long int)xni_target_buffer_target_offset(wdp->wd_e2ep->xni_wd_buf), + (long long int)xni_target_buffer_data_length(wdp->wd_e2ep->xni_wd_buf), wdp->wd_e2ep->e2e_send_status); } if (wdp->wd_current_state & WORKER_CURRENT_STATE_BARRIER) { diff --git a/src/common/debug.c b/src/common/debug.c index 88990543..96f771c6 100644 --- a/src/common/debug.c +++ b/src/common/debug.c @@ -399,8 +399,6 @@ xdd_show_e2e(xint_e2e_t *e2ep) { fprintf(stderr,"\txdd_show_e2e: uint32_t e2e_rnamelen=%d\n",e2ep->e2e_rnamelen); // the length of the source socket name fprintf(stderr,"\txdd_show_e2e: int32_t e2e_current_csd=%d\n",e2ep->e2e_current_csd); // the current csd used by the select call on the destination side fprintf(stderr,"\txdd_show_e2e: int32_t e2e_next_csd=%d\n",e2ep->e2e_next_csd); // The next available csd to use - fprintf(stderr,"\txdd_show_e2e: xdd_e2e_header_t *e2e_hdrp=%p\n",e2ep->e2e_hdrp); // Pointer to the header portion of a packet - if (e2ep->e2e_hdrp) xdd_show_e2e_header(e2ep->e2e_hdrp); fprintf(stderr,"\txdd_show_e2e: unsigned char *e2e_datap=%p\n",e2ep->e2e_datap); // Pointer to the data portion of a packet fprintf(stderr,"\txdd_show_e2e: int32_t e2e_header_size=%d\n",e2ep->e2e_header_size); // Size of the header portion of the buffer fprintf(stderr,"\txdd_show_e2e: int32_t e2e_data_size=%d\n",e2ep->e2e_data_size); // Size of the data portion of the buffer @@ -428,21 +426,6 @@ xdd_show_e2e(xint_e2e_t *e2ep) { fprintf(stderr,"xdd_show_e2e:********* End of E2E Data at 0x%p **********\n",e2ep); } // End of xdd_show_e2e() -/*----------------------------------------------------------------------------*/ -/* xdd_show_e2e_header() - Display values in the specified data structure - */ -void -xdd_show_e2e_header(xdd_e2e_header_t *e2ehp) { - fprintf(stderr,"\nxdd_show_e2e_header:********* Start of E2E Header Data at 0x%p **********\n",e2ehp); - fprintf(stderr,"\t\txdd_show_e2e_header: uint32_t e2eh_magic=0x%8x\n",e2ehp->e2eh_magic); // Magic Number - sanity check - fprintf(stderr,"\t\txdd_show_e2e_header: int32_t pad1\n"); - fprintf(stderr,"\t\txdd_show_e2e_header: int64_t e2eh_sequence_number=%lld\n",(long long int)e2ehp->e2eh_sequence_number); // Sequence number of this operation - fprintf(stderr,"\t\txdd_show_e2e_header: int64_t e2eh_byte_offset=%lld\n",(long long int)e2ehp->e2eh_byte_offset); // Offset relative to the beginning of the file of where this data belongs - fprintf(stderr,"\t\txdd_show_e2e_header: int64_t e2eh_data_length=%lld\n",(long long int)e2ehp->e2eh_data_length); // Length of the user data in bytes for this operation - fprintf(stderr,"\txdd_show_e2e_header:********* End of E2E Header Data at 0x%p **********\n",e2ehp); - -} // End of xdd_show_e2e_header() - /*----------------------------------------------------------------------------*/ /* xdd_show_tot_entry() - Display values in the specified data structure */ diff --git a/src/common/end_to_end.h b/src/common/end_to_end.h index a206b204..4a9f3b73 100644 --- a/src/common/end_to_end.h +++ b/src/common/end_to_end.h @@ -31,14 +31,6 @@ #define HOSTNAMELENGTH 1024 #define E2E_ADDRESS_TABLE_ENTRIES 16 -struct xdd_e2e_header { - uint32_t e2eh_magic; // Magic Number - sanity check - int32_t pad1; - int64_t e2eh_sequence_number; // Sequence number of this operation - int64_t e2eh_byte_offset; // Offset relative to the beginning of the file of where this data belongs - int64_t e2eh_data_length; // Length of the user data in bytes for this operation -}; -typedef struct xdd_e2e_header xdd_e2e_header_t; struct xdd_e2e_address_table_entry { char *address; // Pointer to the ASCII string of the address @@ -100,15 +92,13 @@ struct xint_e2e { uint32_t e2e_rnamelen; // the length of the source socket name int32_t e2e_current_csd; // the current csd used by the select call on the destination side int32_t e2e_next_csd; // The next available csd to use - xdd_e2e_header_t *e2e_hdrp; // Pointer to the header portion of a packet unsigned char *e2e_datap; // Pointer to the data portion of a packet + //TODO: remove e2e_header_size int32_t e2e_header_size; // Size of the header portion of the buffer int32_t e2e_data_size; // Size of the data portion of the buffer int32_t e2e_xfer_size; // Number of bytes per End to End request - size of data buffer plus size of E2E Header int32_t e2e_send_status; // Current Send Status int32_t e2e_recv_status; // Current Recv status -#define XDD_E2E_DATA_READY 0xDADADADA // The magic number that should appear at the beginning of each message indicating data is present -#define XDD_E2E_EOF 0xE0F0E0F0 // The magic number that should appear in a message signaling and End of File int64_t e2e_msg_sequence_number;// The Message Sequence Number of the most recent message sent or to be received int32_t e2e_msg_sent; // The number of messages sent int32_t e2e_msg_recv; // The number of messages received @@ -133,6 +123,7 @@ struct xint_e2e { /* XNI Worker data */ xni_target_buffer_t xni_wd_buf; + int received_eof; // TRUE when the source has signaled EOF to this destination worker }; // End of struct xint_e2e definition typedef struct xint_e2e xint_e2e_t; diff --git a/src/common/xint_prototypes.h b/src/common/xint_prototypes.h index ad13965a..c23606d5 100644 --- a/src/common/xint_prototypes.h +++ b/src/common/xint_prototypes.h @@ -44,7 +44,6 @@ void xdd_show_task(xint_task_t *taskp); void xdd_show_occupant(xdd_occupant_t *op); void xdd_show_target_counters(xint_target_counters_t *tcp); void xdd_show_e2e(xint_e2e_t *e2ep); -void xdd_show_e2e_header(xdd_e2e_header_t *e2ehp); void xdd_show_tot(tot_t *totp); void xdd_show_tot_entry(tot_t *totp, int i); void xdd_show_ts_table(xint_timestamp_t *ts_tablep, int target_number); @@ -52,14 +51,7 @@ void xdd_show_ts_header(xdd_ts_header_t *ts_hdrp, int target_number); void xdd_show_results_data(results_t *rp, char *dumptype, xdd_plan_t *planp); // end_to_end.c -int32_t xdd_e2e_src_send(worker_data_t *wdp); -int32_t xdd_e2e_dest_receive(worker_data_t *wdp); -int32_t xdd_e2e_dest_connection(worker_data_t *wdp); -int32_t xdd_e2e_dest_receive_header(worker_data_t *wdp); -int32_t xdd_e2e_dest_receive_data(worker_data_t *wdp); -int32_t xdd_e2e_dest_receive_error(worker_data_t *wdp); int32_t xdd_e2e_eof_source_side(worker_data_t *wdp); -int32_t xdd_e2e_eof_destination_side(worker_data_t *wdp); // end_to_end_init.c int32_t xdd_e2e_target_init(target_data_t *tdp); diff --git a/src/net/end_to_end.c b/src/net/end_to_end.c index 716db644..6a409816 100644 --- a/src/net/end_to_end.c +++ b/src/net/end_to_end.c @@ -16,529 +16,6 @@ */ #include "xint.h" -/*----------------------------------------------------------------------*/ -/* xdd_e2e_src_send() - send the data from source to destination - * This subroutine will take the message header from the Worker Data Struct - * and update it with: - * - senders Worker Thread number - * - time stamp when this packet is being sent normalized to global time - * The message header is placed after the end of the user data in the - * message buffer (thus making it a trailer rather than a header). - * - * Return values: 0 is good, -1 is bad - * - * The size of the buffer depends on whether it is being used for network - * I/O as in an End-to-end operation. For End-to-End operations, the size - * of the buffer is 1 page larger than for non-End-to-End operations. - * - * For normal (non-E2E operations) the buffer pointers are as follows: - * |<----------- wd_buf_size = N Pages ----------------->| - * +-----------------------------------------------------+ - * | data buffer | - * | transfer size (td_xfer_size) rounded up to N pages | - * |<-wd_bufp | - * |<-task_datap | - * +-----------------------------------------------------+ - * - * For End-to-End operations, the buffer pointers are as follows: - * |<------------------- wd_buf_size = N+1 Pages ------------------------>| - * +----------------+-----------------------------------------------------+ - * |<----1 page---->| transfer size (td_xfer_size) rounded up to N pages | - * |<-wd_bufp |<-task_datap | - * | | E2E | E2E | - * | |<-Header->| data buffer | - * +-----*----------*-----------------------------------------------------+ - * ^ ^ - * ^ +-e2e_datap - * +-e2e_hdrp - */ -int32_t -xdd_e2e_src_send(worker_data_t *wdp) { - target_data_t *tdp; - xint_e2e_t *e2ep; // Pointer to the E2E data struct - xdd_e2e_header_t *e2ehp; // Pointer to the E2E Header - int bytes_sent; // Cumulative number of bytes sent - int send_size; // Number of bytes to send for each call to sendto() - int sento_calls; // Number of times sendto() has been called - int max_xfer; - unsigned char *bufp; - xdd_ts_tte_t *ttep; // Pointer to a time stamp table entry - - - tdp = wdp->wd_tdp; - e2ep = wdp->wd_e2ep; - e2ehp = e2ep->e2e_hdrp; -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_src_send: Target: %d: Worker: %d: ENTER: e2ep=%p: e2ehp=%p: e2e_datap=%p\n",(long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, e2ep, e2ehp, e2ep->e2e_datap); - - // The "task" data structure contains variables relevant to the file-read operation - e2ehp->e2eh_sequence_number = wdp->wd_task.task_op_number; - e2ehp->e2eh_byte_offset = wdp->wd_task.task_byte_offset; - e2ehp->e2eh_data_length = wdp->wd_task.task_xfer_size; - - // The message header for this data packet precedes the data portion - if (tdp->td_ts_table.ts_options & (TS_ON | TS_TRIGGERED)) { - ttep = &tdp->td_ts_table.ts_hdrp->tsh_tte[wdp->wd_ts_entry]; - ttep->tte_net_processor_start = xdd_get_processor(); - } - - // Note: the e2ep->e2e_xfer_size is the size of the data field plus the size of the header - max_xfer = MAXMIT_TCP; - bytes_sent = 0; - bufp = (unsigned char *)e2ehp; - sento_calls = 0; - // The transfer size is the size of the header buffer (not the header struct) - // plus the amount of data in the data portion of the IO buffer. - // For EOF operations the amount of data in the data portion should be zero. - e2ep->e2e_xfer_size = sizeof(xdd_e2e_header_t) + e2ehp->e2eh_data_length; - -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_src_send: Target: %d: Worker: %d: Preparing to send %d bytes: e2ep=%p: e2ehp=%p: e2e_datap=%p: e2e_xfer_size=%d: e2eh_data_length=%lld\n",(long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, e2ep->e2e_xfer_size,e2ep,e2ehp,e2ep->e2e_datap,e2ep->e2e_xfer_size,(long long int)e2ehp->e2eh_data_length); -if (xgp->global_options & GO_DEBUG_E2E) xdd_show_e2e_header((xdd_e2e_header_t *)bufp); - - nclk_now(&wdp->wd_counters.tc_current_net_start_time); - while (bytes_sent < e2ep->e2e_xfer_size) { - send_size = e2ep->e2e_xfer_size - bytes_sent; - if (send_size > max_xfer) - send_size = max_xfer; -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_src_send: Target: %d: Worker: %d: Actually sending %d bytes: e2ep=%p: e2ehp=%p: e2e_datap=%p: bytes_sent=%d: e2ehp+bytes_sent=%p: first 8 bytes=0x%016llx\n",(long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, send_size,e2ep,e2ehp,e2ep->e2e_datap,(int)bytes_sent,bufp, *((unsigned long long int *)bufp)); - e2ep->e2e_send_status = sendto(e2ep->e2e_sd, - bufp, - send_size, - 0, - (struct sockaddr *)&e2ep->e2e_sname, - sizeof(struct sockaddr_in)); - if (e2ep->e2e_send_status <= 0) { - xdd_e2e_err(wdp,"xdd_e2e_src_send","ERROR: error sending HEADER+DATA to destination\n"); - return(-1); - } - bytes_sent += e2ep->e2e_send_status; - bufp += e2ep->e2e_send_status; - sento_calls++; -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_src_send: Target: %d: Worker: %d: Sent %d of %d bytes - %d bytes sent so far: bufp=%p\n",(long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, e2ep->e2e_send_status,e2ep->e2e_xfer_size,bytes_sent,bufp); - } - nclk_now(&wdp->wd_counters.tc_current_net_end_time); - // Time stamp if requested - if (tdp->td_ts_table.ts_options & (TS_ON | TS_TRIGGERED)) { - ttep = &tdp->td_ts_table.ts_hdrp->tsh_tte[wdp->wd_ts_entry]; - ttep->tte_net_xfer_size = e2ep->e2e_xfer_size; - ttep->tte_net_start = wdp->wd_counters.tc_current_net_start_time; - ttep->tte_net_end = wdp->wd_counters.tc_current_net_end_time; - ttep->tte_net_processor_end = xdd_get_processor(); - ttep->tte_net_xfer_calls = sento_calls; - } - - // Calculate the Send/Receive time by the time it took the last sendto() to run - e2ep->e2e_sr_time = (wdp->wd_counters.tc_current_net_end_time - wdp->wd_counters.tc_current_net_start_time); - - if (bytes_sent != e2ep->e2e_xfer_size) { - xdd_e2e_err(wdp,"xdd_e2e_src_send","ERROR: could not send header+data from e2e source\n"); - return(-1); - } - -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_src_send: Target: %d: Worker: %d: EXIT...\n",(long long int)pclk_now(),tdp->td_target_number, wdp->wd_worker_number); - return(0); - -} /* end of xdd_e2e_src_send() */ - -/*----------------------------------------------------------------------*/ -/* xdd_e2e_dest_connection() - Wait for an incoming connection and - * return when it arrives. - * Return values: 0 is good, -1 is bad - * - */ -int32_t -xdd_e2e_dest_connection(worker_data_t *wdp) { - target_data_t *tdp; // Pointer to the Target Data - xint_e2e_t *e2ep; // Pointer to the E2E struct for this worker - - tdp = wdp->wd_tdp; - e2ep = wdp->wd_e2ep; - -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_dest_connection: Target %d Worker: %d: ENTER: e2e_nd=%d: e2e_sd=%d: FD_SETSIZE=%d\n", (long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number,e2ep->e2e_nd,e2ep->e2e_sd,FD_SETSIZE); - - int rc = select(e2ep->e2e_nd, &e2ep->e2e_readset, NULL, NULL, NULL); - assert(rc != -1); - /* Handle all the descriptors that are ready */ - /* There are two type of sockets: the one sd socket and multiple - * client sockets. We first check to see if the sd is in the readset. - * If so, this means that a client is trying to make a new connection - * in which case we need to issue an accept to establish the connection - * and obtain a new Client Socket Descriptor (csd). - */ -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_dest_connection: Target %d Worker: %d: Inside SELECT \n", (long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number); - if (FD_ISSET(e2ep->e2e_sd, &e2ep->e2e_readset)) { /* Process an incoming connection */ - e2ep->e2e_current_csd = e2ep->e2e_next_csd; - - e2ep->e2e_csd[e2ep->e2e_current_csd] = accept(e2ep->e2e_sd, (struct sockaddr *)&e2ep->e2e_rname,&e2ep->e2e_rnamelen); -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_dest_connection: Target %d Worker: %d: connection accepted: sd=%d\n", (long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, e2ep->e2e_sd); - - FD_SET(e2ep->e2e_csd[e2ep->e2e_current_csd], &e2ep->e2e_active); /* Mark this fd as active */ - FD_SET(e2ep->e2e_csd[e2ep->e2e_current_csd], &e2ep->e2e_readset); /* Put in readset so that it gets processed */ - - /* Find the next available csd close to the beginning of the CSD array */ - e2ep->e2e_next_csd = 0; - while (e2ep->e2e_csd[e2ep->e2e_next_csd] != 0) { - e2ep->e2e_next_csd++; - if (e2ep->e2e_next_csd == FD_SETSIZE) { - e2ep->e2e_next_csd = 0; - fprintf(xgp->errout,"\n%s: xdd_e2e_dest_connection: Target %d Worker: %d: ERROR: no csd entries left\n", - xgp->progname, - tdp->td_target_number, - wdp->wd_worker_number); - return(-1); - } - } /* end of WHILE loop that finds the next csd entry */ - } /* End of processing an incoming connection */ - return(0); -} // End of xdd_e2e_dest_connection() - -/*----------------------------------------------------------------------*/ -/* xdd_e2e_dest_receive_header() - Receive E2E Header from the source side - * This subroutine will block until the entire header is received or until - * the connection is broken in which case an error is returned. - * - * Return values: Upon successfully reading the header and validating - * it, the size of the header in bytes is returned to the caller. - * Otherwise, in the event of an error, the status of the recvfrom() - * call is returned to the caller. - * - */ -int32_t -xdd_e2e_dest_receive_header(worker_data_t *wdp) { - target_data_t *tdp; // Pointer to the Target Data - xint_e2e_t *e2ep; // Pointer to the E2E struct for this worker - xdd_e2e_header_t *e2ehp; // Pointer to the E2E Header - int status; // function call status - int bytes_received; // Cumulative number of bytes received if multiple recvfrom() invocations are necessary - int receive_size; // The number of bytes to receive for this invocation of recvfrom() - int max_xfer; // Maximum TCP transmission size - unsigned char *bufp; - xdd_ts_tte_t *ttep; // Pointer to a time stamp table entry - - - tdp = wdp->wd_tdp; - e2ep = wdp->wd_e2ep; - e2ehp = e2ep->e2e_hdrp; - - e2ep->e2e_header_size = sizeof(xdd_e2e_header_t); - e2ep->e2e_xfer_size = e2ep->e2e_header_size; - -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_dest_receive_header: Target %d Worker: %d: ENTER: Waiting to receive %d bytes of header: op# %lld: e2ep=%p: e2ehp=%p: e2e_datap=%p\n", (long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, e2ep->e2e_header_size, (long long int)wdp->wd_task.task_op_number, e2ep, e2ehp, e2ep->e2e_datap ); - - max_xfer = MAXMIT_TCP; - - /* This section will check to see which of the Client Socket Descriptors - * are in the readset. For those csd's that are ready, a recv is issued to - * receive the incoming data. - */ - status = -1; // Default status - bytes_received = 0; - for (e2ep->e2e_current_csd = 0; e2ep->e2e_current_csd < FD_SETSIZE; e2ep->e2e_current_csd++) { // Process all CSDs that are ready - if (FD_ISSET(e2ep->e2e_csd[e2ep->e2e_current_csd], &e2ep->e2e_readset)) { /* Process this csd */ - if (tdp->td_ts_table.ts_options & (TS_ON | TS_TRIGGERED)) { - ttep = &tdp->td_ts_table.ts_hdrp->tsh_tte[wdp->wd_ts_entry]; - ttep->tte_net_processor_start = xdd_get_processor(); - } - - nclk_now(&wdp->wd_counters.tc_current_net_start_time); - - // Read in the E2E Header - bytes_received = 0; - bufp = (unsigned char *)e2ehp; - while (bytes_received < e2ep->e2e_header_size) { - // Calculate the max number of bytes we can receive per call to recvfrom() - receive_size = e2ep->e2e_header_size - bytes_received; - if (receive_size > max_xfer) - receive_size = max_xfer; - - // Issue recvfrom() -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_dest_receive_header: Target: %d: Worker: %d: HEADER: Calling recvfrom bytes_received=%d: receive_size=%d: e2ehp=%p: new_bp=%p\n", (long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, bytes_received, receive_size, e2ehp, bufp); - status = recvfrom(e2ep->e2e_csd[e2ep->e2e_current_csd], - bufp, - receive_size, - 0, - NULL, - NULL); -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_dest_receive_header: Target: %d: Worker: %d: HEADER: Received %d of %d bytes\n", (long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, status, e2ep->e2e_header_size); -if (xgp->global_options & GO_DEBUG_E2E) xdd_show_e2e_header(e2ehp); - - // Check for errors - if (status <= 0) { - e2ep->e2e_recv_status = status; - fprintf(xgp->errout,"\n%s: xdd_e2e_dest_receive_header: Target %d Worker: %d: ERROR RECEIVING HEADER: recv_status=%d, errno=%d\n", - xgp->progname, - tdp->td_target_number, - wdp->wd_worker_number, - e2ep->e2e_recv_status, - errno); - return(status); - } - // Otherwise, figure out how much of the header we read - bytes_received += status; - bufp += status; - } // End of WHILE loop that received incoming data from the source machine -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_dest_receive_header: Target: %d: Worker: %d: HEADER: Got the header... now check to see if the status <%d> is > 0 \n", (long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, status); - } // End of IF stmnt that processes a CSD - } // End of FOR loop that processes all CSDs that were ready - - if (bytes_received != e2ep->e2e_header_size) { - // This is an internal error that should not occur... - fprintf(xgp->errout,"\n%s: xdd_e2e_dest_receive_header: Target %d Worker: %d: INTERNAL ERROR: The number of bytes received <%d> is not equal to the size of the E2E Header <%d>\n", - xgp->progname, - tdp->td_target_number, - wdp->wd_worker_number, - bytes_received, - e2ep->e2e_header_size); - return(-1); - } - if ((e2ehp->e2eh_magic != XDD_E2E_DATA_READY) && (e2ehp->e2eh_magic != XDD_E2E_EOF)) { - // Invalid E2E Header - bad magic number - fprintf(xgp->errout,"\n%s: xdd_e2e_dest_receive_header: Target %d Worker: %d: ERROR: Bad magic number 0x%08x on recv %d - should be either 0x%08x or 0x%08x\n", - xgp->progname, - tdp->td_target_number, - wdp->wd_worker_number, - e2ehp->e2eh_magic, - e2ep->e2e_msg_recv, - XDD_E2E_DATA_READY, - XDD_E2E_EOF); - return(-1); - } - - return(bytes_received); - -} // End of xdd_e2e_dest_receive_header() - -/*----------------------------------------------------------------------*/ -/* xdd_e2e_dest_receive_data() - Receive E2E Data from the source side - * This subroutine will block until the entire data portion is received or - * until the connection is broken in which case an error is returned. - * - * Return values: Upon successfully reading the data and validating - * it, the number of data bytes is returned to the caller. - * Otherwise, in the event of an error, the status of the recvfrom() - * call is returned to the caller. - * - */ -int32_t -xdd_e2e_dest_receive_data(worker_data_t *wdp) { - target_data_t *tdp; // Pointer to the Target Data - xint_e2e_t *e2ep; // Pointer to the E2E struct for this worker - xdd_e2e_header_t *e2ehp; // Pointer to the E2E Header - int status; // function call status - int bytes_received; // Cumulative number of bytes received if multiple recvfrom() invocations are necessary - int receive_size; // The number of bytes to receive for this invocation of recvfrom() - int max_xfer; // Maximum TCP transmission size - unsigned char *bufp; - xdd_ts_tte_t *ttep; // Pointer to a time stamp table entry - - - tdp = wdp->wd_tdp; - e2ep = wdp->wd_e2ep; - e2ehp = e2ep->e2e_hdrp; - - e2ep->e2e_data_size = e2ehp->e2eh_data_length; - e2ep->e2e_xfer_size = e2ep->e2e_data_size; - -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_dest_receive_data: Target %d Worker: %d: ENTER: Waiting to receive %d bytes of DATA: op# %lld: e2ep=%p: e2ehp=%p: e2e_datap=%p\n", (long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, e2ep->e2e_data_size, (long long int)wdp->wd_task.task_op_number, e2ep, e2ehp, e2ep->e2e_datap ); - // The following uses strictly TCP - - max_xfer = MAXMIT_TCP; - - /* This section will check to see which of the Client Socket Descriptors - * are in the readset. For those csd's that are ready, a recv is issued to - * receive the incoming data. - */ - status = -1; // Default status - bytes_received = 0; - for (e2ep->e2e_current_csd = 0; e2ep->e2e_current_csd < FD_SETSIZE; e2ep->e2e_current_csd++) { // Process all CSDs that are ready - if (FD_ISSET(e2ep->e2e_csd[e2ep->e2e_current_csd], &e2ep->e2e_readset)) { /* Process this csd */ - if (tdp->td_ts_table.ts_options & (TS_ON | TS_TRIGGERED)) { - ttep = &tdp->td_ts_table.ts_hdrp->tsh_tte[wdp->wd_ts_entry]; - ttep->tte_net_processor_start = xdd_get_processor(); - } - - // Receive DATA if this was a "DATA" message - e2ep->e2e_data_size = e2ehp->e2eh_data_length; - bytes_received = 0; - bufp = (unsigned char *)e2ep->e2e_datap; -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_dest_receive_data: Target: %d: Worker: %d: OK - IT IS A HEADER SO LETS READ DATA: bytes_received=%d:e2e_data_size=%d \n", (long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, bytes_received,e2ep->e2e_data_size); - while (bytes_received < e2ep->e2e_data_size) { - receive_size = e2ep->e2e_data_size - bytes_received; - if (receive_size > max_xfer) - receive_size = max_xfer; - // Issue recvfrom() -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_dest_receive_data: Target %d Worker: %d: BEFORE RECVFROM DATA: Preparing to receive %d bytes of data, op# %lld, buffer address %p, wd_bufp=%p, wd_task.task_datap=%p, e2e_datap=%p\n", (unsigned long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, receive_size, e2ep->e2e_data_size, (long long int)wdp->wd_task.task_op_number, ((unsigned char *)e2ehp) + bytes_received, wdp->wd_bufp, wdp->wd_task.task_datap, e2ep->e2e_datap); - status = recvfrom(e2ep->e2e_csd[e2ep->e2e_current_csd], - bufp, - receive_size, - 0, - NULL, - NULL); -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_dest_receive_data: Target: %d: Worker: %d: AFTER RECVFROM DATA: Received %d of %d bytes\n", (long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, status, e2ep->e2e_data_size); - - // Check for errors - if (status <= 0) { - e2ep->e2e_recv_status = status; - fprintf(xgp->errout,"\n%s: xdd_e2e_dest_receive_data: Target %d Worker: %d: ERROR RECEIVING HEADER: recv_status=%d, errno=%d\n", - xgp->progname, - tdp->td_target_number, - wdp->wd_worker_number, - e2ep->e2e_recv_status, - errno); - return(status); - } - - // Otherwise, figure out how much data we got and go back for more if necessary - bytes_received += status; - bufp += status; - } // End of WHILE loop that reads the E2E Header - } - - } - - return(bytes_received); - -} // End of xdd_e2e_dest_receive_data() - -/*----------------------------------------------------------------------*/ -/* xdd_e2e_dest_receive_error() - Process an error event during recvfrom() - * Return values: None. - */ -int -xdd_e2e_dest_receive_error(worker_data_t *wdp) { - target_data_t *tdp; // Pointer to the Target Data - xint_e2e_t *e2ep; // Pointer to the E2E struct for this worker - int errno_save; // A copy of the errno - - - tdp = wdp->wd_tdp; - e2ep = wdp->wd_e2ep; - - if (e2ep->e2e_recv_status == 0) { // A status of 0 means that the source side shut down unexpectedly - essentially and Enf-Of-File - fprintf(xgp->errout,"\n%s: xdd_e2e_dest_receive_error: Target %d Worker: %d: ERROR: Connection closed prematurely by Source, op number %lld, location %lld\n", - xgp->progname, - tdp->td_target_number, - wdp->wd_worker_number, - (long long int)wdp->wd_task.task_op_number, - (long long int)wdp->wd_task.task_byte_offset); - } else if (e2ep->e2e_recv_status < 0) { // A status less than 0 indicates some kind of error. - errno_save = errno; - fprintf(xgp->errout,"\n%s: xdd_e2e_dest_receive_error: Target %d Worker: %d: ERROR: recvfrom returned -1, errno %d, op number %lld, location %lld\n", - xgp->progname, - tdp->td_target_number, - wdp->wd_worker_number, - errno, - (long long int)wdp->wd_task.task_op_number, - (long long int)wdp->wd_task.task_byte_offset); - - // Restore the errno and display the reason for the error - errno = errno_save; - perror("Reason"); - } - // At this point we need to clear out this csd and "Deactivate" the socket. - FD_CLR(e2ep->e2e_csd[e2ep->e2e_current_csd], &e2ep->e2e_active); - (void) closesocket(e2ep->e2e_csd[e2ep->e2e_current_csd]); - e2ep->e2e_csd[e2ep->e2e_current_csd] = 0; - return(errno); - -} // End of xdd_e2e_dest_receive_error() - -/*----------------------------------------------------------------------*/ -/* xdd_e2e_dest_receive() - Receive data from the source side - * This subroutine will block until data is received or until the - * connection is broken in which case an error is returned. - * - * Return values: 0 is good, -1 is bad - * - */ -// The receive is done in two parts. -// 1) The first part of the receive reads in the header portion of the packet -// which is normally 1 page. The header contains the number of bytes to -// read in for the data portion of the packet. -// 2) The second part of the receive reads in the data portion of the packet. -// If this is an EOF operation then there is no further data to read. -int32_t -xdd_e2e_dest_receive(worker_data_t *wdp) { - target_data_t *tdp; // Pointer to the Target Data - xint_e2e_t *e2ep; // Pointer to the E2E struct for this worker - xdd_e2e_header_t *e2ehp; // Pointer to the E2E Header - int32_t status; // Status of the call to xdd_e2e_dst_connection() - int header_bytes_received; // Number of header bytes received - int data_bytes_received; // Number of data bytes received - nclk_t e2e_wait_1st_msg_start_time; // This is the time stamp of when the first message arrived - xdd_ts_tte_t *ttep; // Pointer to a time stamp table entry - - - tdp = wdp->wd_tdp; - e2ep = wdp->wd_e2ep; - e2ehp = e2ep->e2e_hdrp; - - nclk_now(&e2e_wait_1st_msg_start_time); - wdp->wd_counters.tc_current_net_start_time = e2e_wait_1st_msg_start_time; - - // Make the connection to the source - status = xdd_e2e_dest_connection(wdp); - if (status) - return(-1); - - // Read in the E2E Header, this will tell us the length of the data portion of this E2E Message - header_bytes_received = xdd_e2e_dest_receive_header(wdp); - if (header_bytes_received <= 0) { - xdd_e2e_dest_receive_error(wdp); - return(-1); - } - - data_bytes_received = 0; - if (e2ehp->e2eh_magic == XDD_E2E_DATA_READY) { - // Read in the data portion of this E2E Message - data_bytes_received = xdd_e2e_dest_receive_data(wdp); - if (data_bytes_received <= 0) { - xdd_e2e_dest_receive_error(wdp); - return(-1); - } - } else { -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_dest_receive: Target %d Worker: %d: Received and EOF\n", (unsigned long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number); - } - - nclk_now(&wdp->wd_counters.tc_current_net_end_time); - - // Keep track of the total number of bytes received so far... - e2ep->e2e_recv_status = header_bytes_received + data_bytes_received; - - // This will record the amount of time that we waited from the time we started until we got the first packet - if (!e2ep->e2e_wait_1st_msg) - e2ep->e2e_wait_1st_msg = wdp->wd_counters.tc_current_net_end_time - e2e_wait_1st_msg_start_time; - - // If this is the first packet received by this Worker Thread then record the *end* of this operation as the - // *start* of this pass. The reason is that the initial recvfrom() may have been issued long before the - // Source side started sending data and we need to ignore that startup delay. - if (wdp->wd_counters.tc_pass_start_time == NCLK_MAX) { // This is an indication that this is the fist recvfrom() that has completed - wdp->wd_counters.tc_pass_start_time = wdp->wd_counters.tc_current_net_end_time; - e2ep->e2e_sr_time = 0; // The first Send/Receive time is zero. - } else { // Calculate the Send/Receive time by the time it took the last recvfrom() to run - e2ep->e2e_sr_time = (wdp->wd_counters.tc_current_net_end_time - wdp->wd_counters.tc_current_net_start_time); - } - - - // If time stamping is on then we need to reset these values - if ((tdp->td_ts_table.ts_options & (TS_ON|TS_TRIGGERED))) { - ttep = &tdp->td_ts_table.ts_hdrp->tsh_tte[wdp->wd_ts_entry]; - ttep->tte_net_start = wdp->wd_counters.tc_current_net_start_time; - ttep->tte_net_end = wdp->wd_counters.tc_current_net_end_time; - ttep->tte_net_processor_end = xdd_get_processor(); - ttep->tte_net_xfer_size = e2ep->e2e_recv_status; - ttep->tte_byte_offset = e2ehp->e2eh_byte_offset; - ttep->tte_disk_xfer_size = e2ehp->e2eh_data_length; - ttep->tte_op_number = e2ehp->e2eh_sequence_number; - if (e2ehp->e2eh_magic == XDD_E2E_EOF) - ttep->tte_op_type = SO_OP_EOF; - else ttep->tte_op_type = SO_OP_WRITE; - } - - e2ep->e2e_readset = e2ep->e2e_active; /* Prepare for the next select */ - - return(0); - -} /* end of xdd_e2e_dest_receive() */ /*----------------------------------------------------------------------*/ /* xdd_e2e_eof_source_side() - End-Of-File processing for Source * Return values: 0 is good, -1 is bad @@ -547,20 +24,11 @@ int32_t xdd_e2e_eof_source_side(worker_data_t *wdp) { target_data_t *tdp; xint_e2e_t *e2ep; // Pointer to the E2E struct for this worker - xdd_e2e_header_t *e2ehp; // Pointer to the E2E Header - int bytes_sent; // Cumulative number of bytes sent if multiple sendto() invocations are necessary - int sendto_calls; // Cumulative number of calls if multiple sendto() invocations are necessary - int send_size; // The number of bytes to send for this invocation of sendto() - int max_xfer; // Maximum TCP transmission size - unsigned char *bufp; - xdd_ts_tte_t *ttep; // Pointer to a time stamp table entry - tdp = wdp->wd_tdp; e2ep = wdp->wd_e2ep; - e2ehp = e2ep->e2e_hdrp; - e2ep->e2e_header_size = sizeof(xdd_e2e_header_t); + e2ep->e2e_header_size = 0; e2ep->e2e_xfer_size = e2ep->e2e_header_size; if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_eof_source_side: Target %d Worker: %d: ENTER: \n", (long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number); @@ -571,70 +39,9 @@ if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e e2ep->e2e_sr_time = 0; return 0; } - - // The following uses strictly TCP - max_xfer = MAXMIT_TCP; - - nclk_now(&wdp->wd_counters.tc_current_net_start_time); - e2ehp->e2eh_sequence_number = (wdp->wd_task.task_op_number + wdp->wd_worker_number); // This is an EOF packet header - e2ehp->e2eh_byte_offset = -1; // NA - e2ehp->e2eh_data_length = 0; // NA - no data being sent other than the header - e2ehp->e2eh_magic = XDD_E2E_EOF; - - if (tdp->td_ts_table.ts_options & (TS_ON | TS_TRIGGERED)) { - ttep = &tdp->td_ts_table.ts_hdrp->tsh_tte[wdp->wd_ts_entry]; - ttep->tte_net_processor_start = xdd_get_processor(); - } - // This will send the E2E Header to the Destination - bytes_sent = 0; - sendto_calls = 0; - bufp = (unsigned char *)e2ehp; - while (bytes_sent < e2ep->e2e_header_size) { - send_size = e2ep->e2e_header_size - bytes_sent; - if (send_size > max_xfer) - send_size = max_xfer; - - bufp += bytes_sent; - e2ep->e2e_send_status = sendto(e2ep->e2e_sd, - bufp, - send_size, - 0, - (struct sockaddr *)&e2ep->e2e_sname, - sizeof(struct sockaddr_in)); - - if (e2ep->e2e_send_status <= 0) { - xdd_e2e_err(wdp,"xdd_e2e_eof_source_side","ERROR: error sending EOF to destination\n"); - return(-1); - } - bytes_sent += e2ep->e2e_send_status; - sendto_calls++; -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_eof_source_side: Target: %d: Worker: %d: Sent %d of %d bytes of the HEADER - %d bytes sent so far\n", (long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, send_size, e2ep->e2e_header_size,bytes_sent); - } - nclk_now(&wdp->wd_counters.tc_current_net_end_time); - - // Calculate the Send/Receive time by the time it took the last sendto() to run - e2ep->e2e_sr_time = (wdp->wd_counters.tc_current_net_end_time - wdp->wd_counters.tc_current_net_start_time); - // If time stamping is on then we need to reset these values - if ((tdp->td_ts_table.ts_options & (TS_ON|TS_TRIGGERED))) { - ttep = &tdp->td_ts_table.ts_hdrp->tsh_tte[wdp->wd_ts_entry]; - ttep->tte_net_start = wdp->wd_counters.tc_current_net_start_time; - ttep->tte_net_end = wdp->wd_counters.tc_current_net_end_time; - ttep->tte_net_processor_end = xdd_get_processor(); - ttep->tte_net_xfer_size = bytes_sent; - ttep->tte_net_xfer_calls = sendto_calls; - ttep->tte_byte_offset = -1; - ttep->tte_disk_xfer_size = 0; - ttep->tte_op_number =e2ehp->e2eh_sequence_number; - ttep->tte_op_type = TASK_OP_TYPE_EOF; - } - - - if (bytes_sent != e2ep->e2e_header_size) { - xdd_e2e_err(wdp,"xdd_e2e_eof_source_side","ERROR: could not send EOF to destination\n"); - return(-1); - } - return(0); + // we only support XNI now + return -1; } /* end of xdd_e2e_eof_source_side() */ /* diff --git a/src/net/end_to_end_init.c b/src/net/end_to_end_init.c index fcf15c10..94a4d561 100644 --- a/src/net/end_to_end_init.c +++ b/src/net/end_to_end_init.c @@ -150,13 +150,7 @@ xdd_e2e_src_init(worker_data_t *wdp) { // Init the relevant variables e2ep->e2e_msg_sent = 0; e2ep->e2e_msg_sequence_number = 0; - e2ep->e2e_header_size = (int)(sizeof(xdd_e2e_header_t)); - - // Init the message header - e2ep->e2e_hdrp->e2eh_sequence_number = 0; - e2ep->e2e_hdrp->e2eh_byte_offset = 0; - e2ep->e2e_hdrp->e2eh_data_length = 0; - + e2ep->e2e_header_size = 0; return(0); @@ -285,6 +279,9 @@ xdd_e2e_dest_init(worker_data_t *wdp) { wdp->wd_e2ep->e2e_msg_recv = 0; wdp->wd_e2ep->e2e_msg_sequence_number = 0; + // Clear the end-of-file flag + wdp->wd_e2ep->received_eof = FALSE; + return(0); } /* end of xdd_e2e_dest_init() */ diff --git a/src/xnet/xnet_end_to_end.c b/src/xnet/xnet_end_to_end.c index b1035b6d..a2d2dc2f 100644 --- a/src/xnet/xnet_end_to_end.c +++ b/src/xnet/xnet_end_to_end.c @@ -138,7 +138,6 @@ int32_t xint_e2e_dest_disconnect(target_data_t *tdp) { int32_t xint_e2e_xni_send(worker_data_t *wdp) { target_data_t *tdp; xint_e2e_t *e2ep; // Pointer to the E2E data struct - xdd_e2e_header_t *e2ehp; // Pointer to the E2E Header //int bytes_sent; // Cumulative number of bytes sent //int sento_calls; // Number of times sendto() has been called xdd_ts_tte_t *ttep; // Pointer to a time stamp table entry @@ -146,9 +145,8 @@ int32_t xint_e2e_xni_send(worker_data_t *wdp) { /* Local aliases */ tdp = wdp->wd_tdp; e2ep = wdp->wd_e2ep; - e2ehp = e2ep->e2e_hdrp; - de2eprintf("DEBUG_E2E: %lld: xdd_e2e_src_send: Target: %d: Worker: %d: ENTER: e2ep=%p: e2ehp=%p: e2e_datap=%p\n",(long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, e2ep, e2ehp, e2ep->e2e_datap); + de2eprintf("DEBUG_E2E: %lld: xdd_e2e_src_send: Target: %d: Worker: %d: ENTER: e2ep=%p: e2e_datap=%p\n",(long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, e2ep, e2ep->e2e_datap); /* Some timestamp code */ if (tdp->td_ts_table.ts_options & (TS_ON | TS_TRIGGERED)) { @@ -156,41 +154,28 @@ int32_t xint_e2e_xni_send(worker_data_t *wdp) { ttep->tte_net_processor_start = xdd_get_processor(); } - /* Construct the e2e header */ - e2ehp->e2eh_sequence_number = wdp->wd_task.task_op_number; - e2ehp->e2eh_byte_offset = wdp->wd_task.task_byte_offset; - e2ehp->e2eh_data_length = wdp->wd_task.task_xfer_size; - e2ep->e2e_xfer_size = sizeof(xdd_e2e_header_t) + e2ehp->e2eh_data_length; - e2ep->e2e_xfer_size = getpagesize() + e2ehp->e2eh_data_length; + // Set XNI parameters and send + xni_target_buffer_set_sequence_number(wdp->wd_task.task_op_number, + wdp->wd_e2ep->xni_wd_buf); + xni_target_buffer_set_target_offset(wdp->wd_task.task_byte_offset, + wdp->wd_e2ep->xni_wd_buf); + xni_target_buffer_set_data_length(wdp->wd_task.task_xfer_size, + wdp->wd_e2ep->xni_wd_buf); + e2ep->e2e_xfer_size = wdp->wd_task.task_xfer_size; - de2eprintf("DEBUG_E2E: %lld: xdd_e2e_src_send: Target: %d: Worker: %d: Preparing to send %d bytes: e2ep=%p: e2ehp=%p: e2e_datap=%p: e2e_xfer_size=%d: e2eh_data_length=%lld\n",(long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, e2ep->e2e_xfer_size,e2ep,e2ehp,e2ep->e2e_datap,e2ep->e2e_xfer_size,(long long int)e2ehp->e2eh_data_length); - if (xgp->global_options & GO_DEBUG_E2E) xdd_show_e2e_header((xdd_e2e_header_t *)xni_target_buffer_data(wdp->wd_e2ep->xni_wd_buf)); + de2eprintf("DEBUG_E2E: %lld: xdd_e2e_src_send: Target: %d: Worker: %d: Preparing to send %d bytes: e2ep=%p: e2e_datap=%p: e2e_xfer_size=%d: e2eh_data_length=%lld\n",(long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, e2ep->e2e_xfer_size,e2ep,e2ep->e2e_datap,e2ep->e2e_xfer_size,(long long int)xni_target_buffer_data_length(wdp->wd_e2ep->xni_wd_buf)); nclk_now(&wdp->wd_counters.tc_current_net_start_time); - /* Don't send magic eof across wire */ - if (XDD_E2E_EOF != e2ehp->e2eh_magic) { - /* Set XNI parameters and send */ - xni_target_buffer_set_sequence_number(e2ehp->e2eh_sequence_number, - e2ep->xni_wd_buf); - xni_target_buffer_set_target_offset(e2ehp->e2eh_byte_offset, - e2ep->xni_wd_buf); - xni_target_buffer_set_data_length(e2ep->e2e_xfer_size, - e2ep->xni_wd_buf); - e2ep->e2e_send_status = xni_send_target_buffer(tdp->td_e2ep->xni_td_conn, - &e2ep->xni_wd_buf); - /* Request a fresh buffer from XNI */ - xni_request_target_buffer(tdp->td_e2ep->xni_td_conn, &wdp->wd_e2ep->xni_wd_buf); - - /* The first page is XNI, the second page is E2E header */ - uintptr_t bufp = - (uintptr_t) xni_target_buffer_data(wdp->wd_e2ep->xni_wd_buf); - wdp->wd_task.task_datap = (unsigned char*)(bufp + getpagesize()); - wdp->wd_e2ep->e2e_hdrp = (xdd_e2e_header_t *)(bufp + (getpagesize() - sizeof(xdd_e2e_header_t))); - wdp->wd_e2ep->e2e_datap = wdp->wd_task.task_datap; - } else { - fprintf(stderr, "Triggered EOF\n"); - e2ep->e2e_send_status = e2ep->e2e_xfer_size; - } + + e2ep->e2e_send_status = xni_send_target_buffer(tdp->td_e2ep->xni_td_conn, + &e2ep->xni_wd_buf); + // Request a fresh buffer from XNI + xni_request_target_buffer(tdp->td_e2ep->xni_td_conn, &wdp->wd_e2ep->xni_wd_buf); + + // Keep a pointer to the data portion of the buffer + wdp->wd_task.task_datap = xni_target_buffer_data(wdp->wd_e2ep->xni_wd_buf); + wdp->wd_e2ep->e2e_datap = wdp->wd_task.task_datap; + nclk_now(&wdp->wd_counters.tc_current_net_end_time); // Time stamp if requested @@ -230,7 +215,6 @@ int32_t xint_e2e_xni_send(worker_data_t *wdp) { int32_t xint_e2e_xni_recv(worker_data_t *wdp) { target_data_t *tdp; // Pointer to the Target Data xint_e2e_t *e2ep; // Pointer to the E2E struct for this worker - xdd_e2e_header_t *e2ehp; // Pointer to the E2E Header int32_t status; // Status of the call to xdd_e2e_dst_connection() nclk_t e2e_wait_1st_msg_start_time; // This is the time stamp of when the first message arrived xdd_ts_tte_t *ttep; // Pointer to a time stamp table entry @@ -246,24 +230,13 @@ int32_t xint_e2e_xni_recv(worker_data_t *wdp) { if (XNI_OK == status) { /* Assemble pointers into the worker's target buffer */ - uintptr_t bufp = - (uintptr_t)xni_target_buffer_data(wdp->wd_e2ep->xni_wd_buf); - //for (int i = 0; i < 256; i++) { - // printf("8 bytes %ld: %d\n", bufp + i, *((int*)bufp + i)); - //} - wdp->wd_task.task_datap = (unsigned char*)(bufp + (1*getpagesize())); - wdp->wd_e2ep->e2e_hdrp = (xdd_e2e_header_t *)(bufp + (1*getpagesize() - sizeof(xdd_e2e_header_t))); + wdp->wd_task.task_datap = xni_target_buffer_data(wdp->wd_e2ep->xni_wd_buf); wdp->wd_e2ep->e2e_datap = wdp->wd_task.task_datap; } else if (XNI_EOF == status) { - /* No buffer set on EOF, so just create a static one */ - xdd_e2e_header_t *eof_header = malloc(sizeof(*eof_header)); - wdp->wd_e2ep->e2e_hdrp = eof_header; - wdp->wd_task.task_datap = 0; - wdp->wd_e2ep->e2e_datap = 0; - - /* Perform EOF Assembly */ - wdp->wd_e2ep->e2e_hdrp->e2eh_magic = XDD_E2E_EOF; + wdp->wd_task.task_datap = NULL; + wdp->wd_e2ep->e2e_datap = NULL; + wdp->wd_e2ep->received_eof = TRUE; } else { fprintf(xgp->errout, "Error receiving data via XNI."); return -1; @@ -275,7 +248,6 @@ int32_t xint_e2e_xni_recv(worker_data_t *wdp) { /* Local aliases */ e2ep = wdp->wd_e2ep; - e2ehp = e2ep->e2e_hdrp; /* Collect the end time */ nclk_now(&wdp->wd_counters.tc_current_net_end_time); @@ -291,12 +263,14 @@ int32_t xint_e2e_xni_recv(worker_data_t *wdp) { ttep->tte_net_end = wdp->wd_counters.tc_current_net_end_time; ttep->tte_net_processor_end = xdd_get_processor(); ttep->tte_net_xfer_size = e2ep->e2e_recv_status; - ttep->tte_byte_offset = e2ehp->e2eh_byte_offset; - ttep->tte_disk_xfer_size = e2ehp->e2eh_data_length; - ttep->tte_op_number = e2ehp->e2eh_sequence_number; - if (e2ehp->e2eh_magic == XDD_E2E_EOF) + ttep->tte_byte_offset = xni_target_buffer_target_offset(e2ep->xni_wd_buf); + ttep->tte_disk_xfer_size = xni_target_buffer_data_length(e2ep->xni_wd_buf); + ttep->tte_op_number = xni_target_buffer_sequence_number(e2ep->xni_wd_buf); + if (wdp->wd_e2ep->received_eof) { ttep->tte_op_type = SO_OP_EOF; - else ttep->tte_op_type = SO_OP_WRITE; + } else { + ttep->tte_op_type = SO_OP_WRITE; + } } return(0); From 033bc09604fb1d69199d5fbf7fe5d913603d4cea Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Wed, 16 Jul 2014 16:00:40 -0400 Subject: [PATCH 27/50] Remove unnecessary fields from E2E structure --- src/base/target_ttd_before_pass.c | 7 - src/base/worker_thread_init.c | 2 - src/base/worker_thread_ttd_after_io_op.c | 1 - src/base/worker_thread_ttd_before_io_op.c | 3 - src/client/interactive_func.c | 5 +- src/client/parse_func.c | 4 - src/common/debug.c | 26 -- src/common/end_to_end.h | 63 ---- src/common/target_data.c | 1 - src/common/xint_prototypes.h | 6 - src/net/end_to_end.c | 3 - src/net/end_to_end_init.c | 345 +--------------------- src/xnet/xnet_end_to_end.c | 21 +- 13 files changed, 11 insertions(+), 476 deletions(-) diff --git a/src/base/target_ttd_before_pass.c b/src/base/target_ttd_before_pass.c index 24ceb0cd..2e0955a5 100644 --- a/src/base/target_ttd_before_pass.c +++ b/src/base/target_ttd_before_pass.c @@ -142,8 +142,6 @@ xdd_raw_before_pass(target_data_t *tdp) { rawp = tdp->td_rawp; // Initialize the read-after-write variables - rawp->raw_msg_sent = 0; - rawp->raw_msg_recv = 0; rawp->raw_msg_last_sequence = 0; rawp->raw_msg.sequence = 0; rawp->raw_prev_loc = 0; @@ -167,12 +165,7 @@ xdd_e2e_before_pass(target_data_t *tdp) { return; // Initialize the read-after-write variables - tdp->td_e2ep->e2e_msg_sent = 0; - tdp->td_e2ep->e2e_msg_recv = 0; tdp->td_e2ep->e2e_msg_sequence_number = 0; - tdp->td_e2ep->e2e_prev_loc = 0; - tdp->td_e2ep->e2e_prev_len = 0; - tdp->td_e2ep->e2e_data_length = 0; tdp->td_e2ep->e2e_sr_time = 0; } // End of xdd_e2e_before_pass() diff --git a/src/base/worker_thread_init.c b/src/base/worker_thread_init.c index aeafaf40..5f1c66a1 100644 --- a/src/base/worker_thread_init.c +++ b/src/base/worker_thread_init.c @@ -109,7 +109,6 @@ xdd_worker_thread_init(worker_data_t *wdp) { // Buffer for destination is set after a receive wdp->wd_e2ep->xni_wd_buf = NULL; wdp->wd_task.task_datap = NULL; - wdp->wd_e2ep->e2e_datap = NULL; } else { //TODO: I'd really like to move all of these buffer // request calls to something like ttd_before_io_op, but I @@ -120,7 +119,6 @@ xdd_worker_thread_init(worker_data_t *wdp) { // Request an I/O buffer from XNI xni_request_target_buffer(tdp->td_e2ep->xni_td_conn, &wdp->wd_e2ep->xni_wd_buf); wdp->wd_task.task_datap = xni_target_buffer_data(wdp->wd_e2ep->xni_wd_buf); - wdp->wd_e2ep->e2e_datap = wdp->wd_task.task_datap; } } else { // For non-E2E operations the data portion is the entire buffer diff --git a/src/base/worker_thread_ttd_after_io_op.c b/src/base/worker_thread_ttd_after_io_op.c index 898a8c39..02e5411a 100644 --- a/src/base/worker_thread_ttd_after_io_op.c +++ b/src/base/worker_thread_ttd_after_io_op.c @@ -234,7 +234,6 @@ if (xgp->global_options & GO_DEBUG_E2E) xdd_show_task(&wdp->wd_task); // Release the current target buffer to XNI xni_release_target_buffer(&wdp->wd_e2ep->xni_wd_buf); wdp->wd_task.task_datap = NULL; - wdp->wd_e2ep->e2e_datap = NULL; } if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_after_io_op: Target: %d: Worker: %d: EXIT...\n", (long long int)pclk_now(),tdp->td_target_number,wdp->wd_worker_number); diff --git a/src/base/worker_thread_ttd_before_io_op.c b/src/base/worker_thread_ttd_before_io_op.c index a09795bf..971f7646 100644 --- a/src/base/worker_thread_ttd_before_io_op.c +++ b/src/base/worker_thread_ttd_before_io_op.c @@ -201,7 +201,6 @@ xdd_e2e_before_io_op(worker_data_t *wdp) { /* ------------------------------------------------------ */ // We are the Destination side of an End-to-End op - wdp->wd_e2ep->e2e_data_recvd = 0; // This will record how much data is recvd in this routine // Lets read a packet of data from the Source side // The call to xint_e2e_xni_recv() will block until there is data to read @@ -225,8 +224,6 @@ xdd_e2e_before_io_op(worker_data_t *wdp) { wdp->wd_task.task_byte_offset = xni_target_buffer_target_offset(wdp->wd_e2ep->xni_wd_buf); wdp->wd_task.task_xfer_size = xni_target_buffer_data_length(wdp->wd_e2ep->xni_wd_buf); wdp->wd_task.task_op_number = xni_target_buffer_sequence_number(wdp->wd_e2ep->xni_wd_buf); - // Record the amount of data received - wdp->wd_e2ep->e2e_data_recvd = xni_target_buffer_data_length(wdp->wd_e2ep->xni_wd_buf); return(0); diff --git a/src/client/interactive_func.c b/src/client/interactive_func.c index 2987bddd..a48272ac 100644 --- a/src/client/interactive_func.c +++ b/src/client/interactive_func.c @@ -321,11 +321,10 @@ xdd_interactive_display_state_info(worker_data_t *wdp) { (long long int)wdp->wd_counters.tc_current_op_end_time); } if (wdp->wd_current_state & WORKER_CURRENT_STATE_DEST_RECEIVE) { - fprintf(xgp->output," Destination Side of an E2E - waiting to receive data from Source, target op number %lld, location %lld, length %lld, recvfrom status is %d\n", + fprintf(xgp->output," Destination Side of an E2E - waiting to receive data from Source, target op number %lld, location %lld, length %lld\n", (long long int)wdp->wd_counters.tc_current_op_number, (long long int)xni_target_buffer_target_offset(wdp->wd_e2ep->xni_wd_buf), - (long long int)xni_target_buffer_data_length(wdp->wd_e2ep->xni_wd_buf), - wdp->wd_e2ep->e2e_recv_status); + (long long int)xni_target_buffer_data_length(wdp->wd_e2ep->xni_wd_buf)); } if (wdp->wd_current_state & WORKER_CURRENT_STATE_SRC_SEND) { fprintf(xgp->output," Source Side of an E2E - waiting to send data to Destination, target op number %lld, location %lld, length %lld, sendto status is %d\n", diff --git a/src/client/parse_func.c b/src/client/parse_func.c index 111428b8..35a05c44 100644 --- a/src/client/parse_func.c +++ b/src/client/parse_func.c @@ -1243,8 +1243,6 @@ xddfunc_endtoend(xdd_plan_t *planp, int32_t argc, char *argv[], uint32_t flags) tdp->td_target_options |= TO_ENDTOEND; source_path = argv[args_index+1]; source_mtime = atoll(argv[args_index+2]); - tdp->td_e2ep->e2e_src_file_path = source_path; - tdp->td_e2ep->e2e_src_file_mtime = source_mtime; } else { /* set option for all targets */ if (flags & XDD_PARSE_PHASE2) { tdp = planp->target_datap[0]; @@ -1253,8 +1251,6 @@ xddfunc_endtoend(xdd_plan_t *planp, int32_t argc, char *argv[], uint32_t flags) source_mtime = atoll(argv[args_index+2]); while (tdp) { tdp->td_target_options |= TO_ENDTOEND; - tdp->td_e2ep->e2e_src_file_path = source_path; - tdp->td_e2ep->e2e_src_file_mtime = source_mtime; i++; tdp = planp->target_datap[i]; } diff --git a/src/common/debug.c b/src/common/debug.c index 96f771c6..a126cb76 100644 --- a/src/common/debug.c +++ b/src/common/debug.c @@ -388,36 +388,10 @@ xdd_show_e2e(xint_e2e_t *e2ep) { fprintf(stderr,"\txdd_show_e2e: time_t e2e_src_file_mtime\n"); // stat -c %Y *e2e_src_file_path, i.e., last modification time fprintf(stderr,"\txdd_show_e2e: in_addr_t e2e_dest_addr=%d\n",e2ep->e2e_dest_addr); // Destination Address number of the E2E socket fprintf(stderr,"\txdd_show_e2e: in_port_t e2e_dest_port=%d\n",e2ep->e2e_dest_port); // Port number to use for the E2E socket - fprintf(stderr,"\txdd_show_e2e: int32_t e2e_sd=%d\n",e2ep->e2e_sd); // Socket descriptor for the E2E message port - fprintf(stderr,"\txdd_show_e2e: int32_t e2e_nd=%d\n",e2ep->e2e_nd); // Number of Socket descriptors in the read set - fprintf(stderr,"\txdd_show_e2e: sd_t e2e_csd[FD_SETSIZE]\n");; // Client socket descriptors - fprintf(stderr,"\txdd_show_e2e: fd_set e2e_active?\n"); // This set contains the sockets currently active - fprintf(stderr,"\txdd_show_e2e: fd_set e2e_readset?\n"); // This set is passed to select() - fprintf(stderr,"\txdd_show_e2e: struct sockaddr_in e2e_sname?\n"); // used by setup_server_socket - fprintf(stderr,"\txdd_show_e2e: uint32_t e2e_snamelen=%d\n",e2ep->e2e_snamelen); // the length of the socket name - fprintf(stderr,"\txdd_show_e2e: struct sockaddr_in e2e_rname?\n"); // used by destination machine to remember the name of the source machine - fprintf(stderr,"\txdd_show_e2e: uint32_t e2e_rnamelen=%d\n",e2ep->e2e_rnamelen); // the length of the source socket name - fprintf(stderr,"\txdd_show_e2e: int32_t e2e_current_csd=%d\n",e2ep->e2e_current_csd); // the current csd used by the select call on the destination side - fprintf(stderr,"\txdd_show_e2e: int32_t e2e_next_csd=%d\n",e2ep->e2e_next_csd); // The next available csd to use - fprintf(stderr,"\txdd_show_e2e: unsigned char *e2e_datap=%p\n",e2ep->e2e_datap); // Pointer to the data portion of a packet - fprintf(stderr,"\txdd_show_e2e: int32_t e2e_header_size=%d\n",e2ep->e2e_header_size); // Size of the header portion of the buffer - fprintf(stderr,"\txdd_show_e2e: int32_t e2e_data_size=%d\n",e2ep->e2e_data_size); // Size of the data portion of the buffer - fprintf(stderr,"\txdd_show_e2e: int32_t e2e_xfer_size=%d\n",e2ep->e2e_xfer_size); // Number of bytes per End to End request - size of data buffer plus size of E2E Header fprintf(stderr,"\txdd_show_e2e: int32_t e2e_send_status=%d\n",e2ep->e2e_send_status); // Current Send Status - fprintf(stderr,"\txdd_show_e2e: int32_t e2e_recv_status=%d\n",e2ep->e2e_recv_status); // Current Recv status fprintf(stderr,"\txdd_show_e2e: int64_t e2e_msg_sequence_number=%lld\n",(long long int)e2ep->e2e_msg_sequence_number);// The Message Sequence Number of the most recent message sent or to be received - fprintf(stderr,"\txdd_show_e2e: int32_t e2e_msg_sent=%d\n",e2ep->e2e_msg_sent); // The number of messages sent - fprintf(stderr,"\txdd_show_e2e: int32_t e2e_msg_recv=%d\n",e2ep->e2e_msg_recv); // The number of messages received - fprintf(stderr,"\txdd_show_e2e: int64_t e2e_prev_loc=%lld\n",(long long int)e2ep->e2e_prev_loc); // The previous location from a e2e message from the source - fprintf(stderr,"\txdd_show_e2e: int64_t e2e_prev_len=%lld\n",(long long int)e2ep->e2e_prev_len); // The previous length from a e2e message from the source - fprintf(stderr,"\txdd_show_e2e: int64_t e2e_data_recvd=%lld\n",(long long int)e2ep->e2e_data_recvd); // The amount of data that is received each time we call xdd_e2e_dest_recv() - fprintf(stderr,"\txdd_show_e2e: int64_t e2e_data_length=%lld\n",(long long int)e2ep->e2e_data_length); // The amount of data that is ready to be read for this operation fprintf(stderr,"\txdd_show_e2e: int64_t e2e_total_bytes_written=%lld\n",(long long int)e2ep->e2e_total_bytes_written); // The total amount of data written across all restarts for this file fprintf(stderr,"\txdd_show_e2e: nclk_t e2e_wait_1st_msg=%lld\n",(unsigned long long int)e2ep->e2e_wait_1st_msg); // Time in nanosecs destination waited for 1st source data to arrive - fprintf(stderr,"\txdd_show_e2e: nclk_t e2e_first_packet_received_this_pass=%lld\n",(unsigned long long int)e2ep->e2e_first_packet_received_this_pass);// Time that the first packet was received by the destination from the source - fprintf(stderr,"\txdd_show_e2e: nclk_t e2e_last_packet_received_this_pass=%lld\n",(unsigned long long int)e2ep->e2e_last_packet_received_this_pass);// Time that the last packet was received by the destination from the source - fprintf(stderr,"\txdd_show_e2e: nclk_t e2e_first_packet_received_this_run=%lld\n",(unsigned long long int)e2ep->e2e_first_packet_received_this_run);// Time that the first packet was received by the destination from the source - fprintf(stderr,"\txdd_show_e2e: nclk_t e2e_last_packet_received_this_run=%lld\n",(unsigned long long int)e2ep->e2e_last_packet_received_this_run);// Time that the last packet was received by the destination from the source fprintf(stderr,"\txdd_show_e2e: nclk_t e2e_sr_time=%lld\n",(unsigned long long int)e2ep->e2e_sr_time); // Time spent sending or receiving data for End-to-End operation fprintf(stderr,"\txdd_show_e2e: int32_t e2e_address_table_host_count=%d\n",e2ep->e2e_address_table_host_count); // Cumulative number of hosts represented in the e2e address table fprintf(stderr,"\txdd_show_e2e: int32_t e2e_address_table_port_count=%d\n",e2ep->e2e_address_table_port_count); // Cumulative number of ports represented in the e2e address table diff --git a/src/common/end_to_end.h b/src/common/end_to_end.h index 4a9f3b73..f27889b1 100644 --- a/src/common/end_to_end.h +++ b/src/common/end_to_end.h @@ -10,18 +10,6 @@ * Foundation. See file COPYING. * */ -/* - * +-----+-----------------------------------------------+ - * | hdr | Data | - * +-----+-----------------------------------------------+ - * - * +----//----------------------+----------------------------------------+ - * | | E2E Header | Data ---- N bytes ---- | - * | |<--64 bytes-->|<< Start of data buffer is Page Aligned | - * |<---//----PAGE_SIZE bytes-->| | - * +----//----------------------+----------------------------------------+ - * - */ #ifdef HAVE_SCHED_H #include @@ -48,28 +36,6 @@ struct xdd_e2e_address_table { }; typedef struct xdd_e2e_address_table xdd_e2e_at_t; -// Things used in the various end_to_end subroutines. -#ifdef FD_SETSIZE -#undef FD_SETSIZE -#define FD_SETSIZE 128 -#endif - -#define MAXMIT_TCP (1<<28) - -/* - * The xint_td_e2e structure contains variables that are referenced by the - * target thread. - */ -struct xint_td_e2e { - char *e2e_dest_hostname; // Name of the Destination machine - char *e2e_src_hostname; // Name of the Source machine - char *e2e_src_file_path; // Full path of source file for destination restart file - time_t e2e_src_file_mtime; // stat -c %Y *e2e_src_file_path, i.e., last modification time - in_addr_t e2e_dest_addr; // Destination Address number of the E2E socket - in_port_t e2e_dest_port; // Port number to use for the E2E socket -}; -typedef struct xint_td_e2e xint_td_e2e_t; - /* * The xint_e2e structure contains variables that are referenced by the * target thread and worker thread. @@ -77,41 +43,12 @@ typedef struct xint_td_e2e xint_td_e2e_t; struct xint_e2e { char *e2e_dest_hostname; // Name of the Destination machine char *e2e_src_hostname; // Name of the Source machine - char *e2e_src_file_path; // Full path of source file for destination restart file - time_t e2e_src_file_mtime; // stat -c %Y *e2e_src_file_path, i.e., last modification time in_addr_t e2e_dest_addr; // Destination Address number of the E2E socket in_port_t e2e_dest_port; // Port number to use for the E2E socket - int32_t e2e_sd; // Socket descriptor for the E2E message port - int32_t e2e_nd; // Number of Socket descriptors in the read set - sd_t e2e_csd[FD_SETSIZE]; // Client socket descriptors - fd_set e2e_active; // This set contains the sockets currently active - fd_set e2e_readset; // This set is passed to select() - struct sockaddr_in e2e_sname; // used by setup_server_socket - uint32_t e2e_snamelen; // the length of the socket name - struct sockaddr_in e2e_rname; // used by destination machine to remember the name of the source machine - uint32_t e2e_rnamelen; // the length of the source socket name - int32_t e2e_current_csd; // the current csd used by the select call on the destination side - int32_t e2e_next_csd; // The next available csd to use - unsigned char *e2e_datap; // Pointer to the data portion of a packet - //TODO: remove e2e_header_size - int32_t e2e_header_size; // Size of the header portion of the buffer - int32_t e2e_data_size; // Size of the data portion of the buffer - int32_t e2e_xfer_size; // Number of bytes per End to End request - size of data buffer plus size of E2E Header int32_t e2e_send_status; // Current Send Status - int32_t e2e_recv_status; // Current Recv status int64_t e2e_msg_sequence_number;// The Message Sequence Number of the most recent message sent or to be received - int32_t e2e_msg_sent; // The number of messages sent - int32_t e2e_msg_recv; // The number of messages received - int64_t e2e_prev_loc; // The previous location from a e2e message from the source - int64_t e2e_prev_len; // The previous length from a e2e message from the source - int64_t e2e_data_recvd; // The amount of data that is received each time we call xdd_e2e_dest_recv() - int64_t e2e_data_length; // The amount of data that is ready to be read for this operation int64_t e2e_total_bytes_written; // The total amount of data written across all restarts for this file nclk_t e2e_wait_1st_msg; // Time in nanosecs destination waited for 1st source data to arrive - nclk_t e2e_first_packet_received_this_pass;// Time that the first packet was received by the destination from the source - nclk_t e2e_last_packet_received_this_pass;// Time that the last packet was received by the destination from the source - nclk_t e2e_first_packet_received_this_run;// Time that the first packet was received by the destination from the source - nclk_t e2e_last_packet_received_this_run;// Time that the last packet was received by the destination from the source nclk_t e2e_sr_time; // Time spent sending or receiving data for End-to-End operation int32_t e2e_address_table_host_count; // Cumulative number of hosts represented in the e2e address table int32_t e2e_address_table_port_count; // Cumulative number of ports represented in the e2e address table diff --git a/src/common/target_data.c b/src/common/target_data.c index 215b10e3..f3551fa0 100644 --- a/src/common/target_data.c +++ b/src/common/target_data.c @@ -94,7 +94,6 @@ xdd_init_new_target_data(target_data_t *tdp, int32_t n) { } /* Init the end-to-end fields */ if (tdp->td_e2ep) { - tdp->td_e2ep->e2e_sd = 0; /* destination machine socket descriptor */ tdp->td_e2ep->e2e_src_hostname = NULL; /* E2E source hostname */ tdp->td_e2ep->e2e_dest_hostname = NULL; /* E2E destination hostname */ tdp->td_e2ep->e2e_dest_port = DEFAULT_E2E_PORT; diff --git a/src/common/xint_prototypes.h b/src/common/xint_prototypes.h index c23606d5..86d4fa76 100644 --- a/src/common/xint_prototypes.h +++ b/src/common/xint_prototypes.h @@ -57,13 +57,7 @@ int32_t xdd_e2e_eof_source_side(worker_data_t *wdp); int32_t xdd_e2e_target_init(target_data_t *tdp); int32_t xdd_e2e_worker_init(worker_data_t *wdp); int32_t xdd_e2e_src_init(worker_data_t *wdp); -int32_t xdd_e2e_setup_src_socket(worker_data_t *wdp); int32_t xdd_e2e_dest_init(worker_data_t *wdp); -int32_t xdd_e2e_setup_dest_socket(worker_data_t *wdp); -void xdd_e2e_set_socket_opts(worker_data_t *wdp, int skt); -void xdd_e2e_prt_socket_opts(int skt); -void xdd_e2e_err(worker_data_t *wdp, char const *whence, char const *fmt, ...); -int32_t xdd_sockets_init(void); // global_clock.c in_addr_t xdd_init_global_clock_network(char *hostname); diff --git a/src/net/end_to_end.c b/src/net/end_to_end.c index 6a409816..cbdfef94 100644 --- a/src/net/end_to_end.c +++ b/src/net/end_to_end.c @@ -28,9 +28,6 @@ xdd_e2e_eof_source_side(worker_data_t *wdp) { tdp = wdp->wd_tdp; e2ep = wdp->wd_e2ep; - e2ep->e2e_header_size = 0; - e2ep->e2e_xfer_size = e2ep->e2e_header_size; - if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_eof_source_side: Target %d Worker: %d: ENTER: \n", (long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number); /* If this is XNI, just short circuit */ diff --git a/src/net/end_to_end_init.c b/src/net/end_to_end_init.c index 94a4d561..1192af22 100644 --- a/src/net/end_to_end_init.c +++ b/src/net/end_to_end_init.c @@ -30,18 +30,10 @@ xdd_e2e_target_init(target_data_t *tdp) { int status; // Perform XNI initialization if required - xdd_plan_t *planp = tdp->td_planp; - if (PLAN_ENABLE_XNI & planp->plan_options) { - xint_e2e_xni_init(tdp); - } - else { - - // Init the sockets - This is actually just for Windows that requires some additional initting - status = xdd_sockets_init(); - if (status == -1) { - fprintf(xgp->errout,"%s: xdd_e2e_target_init: could not initialize sockets for e2e target\n",xgp->progname); - return(-1); - } + status = xint_e2e_xni_init(tdp); + if (status == -1) { + fprintf(xgp->errout,"%s: xdd_e2e_target_init: could not initialize XNI for e2e target\n",xgp->progname); + return(-1); } // Restart processing if necessary @@ -120,7 +112,6 @@ int32_t xdd_e2e_src_init(worker_data_t *wdp) { target_data_t *tdp; xint_e2e_t *e2ep; // Pointer to the E2E data struct - int status; // status of various function calls tdp = wdp->wd_tdp; @@ -137,89 +128,13 @@ xdd_e2e_src_init(worker_data_t *wdp) { return(-1); } - /* Only setup sockets if not using XNI */ - xdd_plan_t *planp = tdp->td_planp; - if (!(PLAN_ENABLE_XNI & planp->plan_options)) { - status = xdd_e2e_setup_src_socket(wdp); - if (status == -1){ - xdd_e2e_err(wdp,"xdd_e2e_src_init","could not setup sockets for e2e source\n"); - return(-1); - } - } - // Init the relevant variables - e2ep->e2e_msg_sent = 0; e2ep->e2e_msg_sequence_number = 0; - e2ep->e2e_header_size = 0; return(0); } /* end of xdd_e2e_src_init() */ -/*----------------------------------------------------------------------*/ -/* xdd_e2e_setup_src_socket() - set up the source side - * This subroutine is called by xdd_e2e_src_init() and is passed a - * pointer to the Data Struct of the requesting Worker Thread. - * - * Return values: 0 is good, -1 is bad - * - */ -int32_t -xdd_e2e_setup_src_socket(worker_data_t *wdp) { - int status; /* status of send/recv function calls */ - int type; - static const int connect_try_limit = 4; - int i; - - // The socket type is SOCK_STREAM because this is a TCP connection - type = SOCK_STREAM; - - // Create the socket - wdp->wd_e2ep->e2e_sd = socket(AF_INET, type, IPPROTO_TCP); - if (wdp->wd_e2ep->e2e_sd < 0) { - xdd_e2e_err(wdp,"xdd_e2e_setup_src_socket","ERROR: error openning socket\n"); - return(-1); - } - (void) xdd_e2e_set_socket_opts (wdp,wdp->wd_e2ep->e2e_sd); - - /* Now build the "name" of the DESTINATION machine socket thingy and connect to it. */ - (void) memset(&wdp->wd_e2ep->e2e_sname, 0, sizeof(wdp->wd_e2ep->e2e_sname)); - wdp->wd_e2ep->e2e_sname.sin_family = AF_INET; - wdp->wd_e2ep->e2e_sname.sin_addr.s_addr = htonl(wdp->wd_e2ep->e2e_dest_addr); - wdp->wd_e2ep->e2e_sname.sin_port = htons(wdp->wd_e2ep->e2e_dest_port); - wdp->wd_e2ep->e2e_snamelen = sizeof(wdp->wd_e2ep->e2e_sname); - - // Attempt to connect to the server for roughly 10 seconds - i = 0; - status = 1; - while (i < connect_try_limit && 0 != status) { - - /* If this is a retry, sleep for 3 seconds before retrying */ - if (i > 0) { - struct timespec req; - memset(&req, 0, sizeof(req)); - req.tv_sec = 3; - fprintf(xgp->errout, - "Socket connection error, retrying in %d seconds: %d\n", - (int)req.tv_sec, status); - nanosleep(&req, (struct timespec *)NULL); - } - - status = connect(wdp->wd_e2ep->e2e_sd, - (struct sockaddr *) &wdp->wd_e2ep->e2e_sname, - sizeof(wdp->wd_e2ep->e2e_sname)); - i++; - } - - if (0 != status) { - xdd_e2e_err(wdp,"xdd_e2e_setup_src_socket","error connecting to socket for E2E destination\n"); - return(-1); - } - - return(0); - -} /* end of xdd_e2e_setup_src_socket() */ - /*----------------------------------------------------------------------*/ /* xdd_e2e_dest_init() - init the destination side * This routine is called by a Worker Thread on the "destination" side of an @@ -232,7 +147,6 @@ xdd_e2e_setup_src_socket(worker_data_t *wdp) { int32_t xdd_e2e_dest_init(worker_data_t *wdp) { target_data_t *tdp; - int status; // status of various function calls tdp = wdp->wd_tdp; @@ -246,37 +160,8 @@ xdd_e2e_dest_init(worker_data_t *wdp) { tdp->td_rwratio); return(-1); } - - /* Only setup sockets if not using XNI */ - xdd_plan_t *planp = tdp->td_planp; - if (!(PLAN_ENABLE_XNI & planp->plan_options)) { - status = xdd_e2e_setup_dest_socket(wdp); - if (status == -1){ - xdd_e2e_err(wdp,"xdd_e2e_dest_init","could not setup sockets for e2e destination\n"); - return(-1); - } - - // Set up the descriptor table for the select() call - // This section is used when we are using TCP - /* clear out the csd table */ - for (wdp->wd_e2ep->e2e_current_csd = 0; wdp->wd_e2ep->e2e_current_csd < FD_SETSIZE; wdp->wd_e2ep->e2e_current_csd++) - wdp->wd_e2ep->e2e_csd[wdp->wd_e2ep->e2e_current_csd] = 0; - - // Set the current and next csd indices to 0 - wdp->wd_e2ep->e2e_current_csd = wdp->wd_e2ep->e2e_next_csd = 0; - - /* Initialize the socket sets for select() */ - FD_ZERO(&wdp->wd_e2ep->e2e_readset); - FD_SET(wdp->wd_e2ep->e2e_sd, &wdp->wd_e2ep->e2e_readset); - wdp->wd_e2ep->e2e_active = wdp->wd_e2ep->e2e_readset; - wdp->wd_e2ep->e2e_current_csd = wdp->wd_e2ep->e2e_next_csd = 0; - - /* Find out how many sockets are in each set */ - wdp->wd_e2ep->e2e_nd = FD_SETSIZE; - } - + // Initialize the message counter and sequencer to 0 - wdp->wd_e2ep->e2e_msg_recv = 0; wdp->wd_e2ep->e2e_msg_sequence_number = 0; // Clear the end-of-file flag @@ -286,226 +171,6 @@ xdd_e2e_dest_init(worker_data_t *wdp) { } /* end of xdd_e2e_dest_init() */ -/*----------------------------------------------------------------------*/ -/* xdd_e2e_setup_dest_socket() - Set up the socket on the Destination side - * This subroutine is called by xdd_e2e_dest_init() and is passed a - * pointer to the Data Struct of the requesting Worker Thread. - * - * Return values: 0 is good, -1 is bad - * - */ -int32_t -xdd_e2e_setup_dest_socket(worker_data_t *wdp) { - int status; - int type; - char msg[256]; - - - // Set the "type" of socket being requested: for TCP, type=SOCK_STREAM - type = SOCK_STREAM; - - // Create the socket - wdp->wd_e2ep->e2e_sd = socket(AF_INET, type, IPPROTO_TCP); - if (wdp->wd_e2ep->e2e_sd < 0) { - xdd_e2e_err(wdp,"xdd_e2e_setup_dest_socket","ERROR: error openning socket\n"); - return(-1); - } - - (void) xdd_e2e_set_socket_opts (wdp, wdp->wd_e2ep->e2e_sd); - - /* Bind the name to the socket */ - (void) memset(&wdp->wd_e2ep->e2e_sname, 0, sizeof(wdp->wd_e2ep->e2e_sname)); - wdp->wd_e2ep->e2e_sname.sin_family = AF_INET; - wdp->wd_e2ep->e2e_sname.sin_addr.s_addr = htonl(wdp->wd_e2ep->e2e_dest_addr); - wdp->wd_e2ep->e2e_sname.sin_port = htons(wdp->wd_e2ep->e2e_dest_port); - wdp->wd_e2ep->e2e_snamelen = sizeof(wdp->wd_e2ep->e2e_sname); - if (bind(wdp->wd_e2ep->e2e_sd, (struct sockaddr *) &wdp->wd_e2ep->e2e_sname, wdp->wd_e2ep->e2e_snamelen)) { - sprintf(msg,"Error binding name to socket - addr=0x%08x, port=0x%08x, specified as %d \n", - wdp->wd_e2ep->e2e_sname.sin_addr.s_addr, - wdp->wd_e2ep->e2e_sname.sin_port, - wdp->wd_e2ep->e2e_dest_port); - xdd_e2e_err(wdp,"xdd_e2e_setup_dest_socket",msg); - return(-1); - } - - /* All set; prepare to accept connection requests */ - if (type == SOCK_STREAM) { // If this is a stream socket then we need to listen for incoming data - status = listen(wdp->wd_e2ep->e2e_sd, SOMAXCONN); - if (status) { - xdd_e2e_err(wdp,"xdd_e2e_setup_dest_socket","ERROR: bad status starting LISTEN on socket\n"); - return(-1); - } - } - - return(0); - -} /* end of xdd_e2e_setup_dest_socket() */ - -/*----------------------------------------------------------------------*/ -/* - * xdd_e2e_set_socket_opts() - set the options for specified socket. - * - */ -void -xdd_e2e_set_socket_opts(worker_data_t *wdp, int skt) { - target_data_t *tdp; - int status; - int level = SOL_SOCKET; - xdd_plan_t* planp = wdp->wd_tdp->td_planp; - -#if WIN32 - char optionvalue; -#else - int optionvalue; -#endif - - tdp = wdp->wd_tdp; - /* Create the socket and set some options */ - optionvalue = 1; - status = setsockopt(skt, IPPROTO_TCP, TCP_NODELAY, &optionvalue, sizeof(optionvalue)); - if (status != 0) { - xdd_e2e_err(wdp,"xdd_e2e_set_socket_opts","Error setting TCP_NODELAY \n"); - } - status = setsockopt(skt,level,SO_SNDBUF,(char *)&planp->e2e_TCP_Win,sizeof(planp->e2e_TCP_Win)); - if (status < 0) { - fprintf(xgp->errout,"%s: xdd_e2e_set_socket_opts: Target %d Worker Thread %d: WARNING: on setsockopt SO_SNDBUF: status %d: %s\n", - xgp->progname, - tdp->td_target_number, - wdp->wd_worker_number, - status, - strerror(errno)); - } - status = setsockopt(skt,level,SO_RCVBUF,(char *)&planp->e2e_TCP_Win,sizeof(planp->e2e_TCP_Win)); - if (status < 0) { - fprintf(xgp->errout,"%s: xdd_e2e_set_socket_opts: Target %d Worker Thread %d: WARNING: on setsockopt SO_RCVBUF: status %d: %s\n", - xgp->progname, - tdp->td_target_number, - wdp->wd_worker_number, - status, - strerror(errno)); - } - status = setsockopt(skt,level,SO_REUSEADDR,(char *)&planp->e2e_TCP_Win,sizeof(planp->e2e_TCP_Win)); - if (status < 0) { - fprintf(xgp->errout,"%s: xdd_e2e_set_socket_opts: Target %d Worker Thread %d: WARNING: on setsockopt SO_REUSEPORT: status %d: %s\n", - xgp->progname, - tdp->td_target_number, - wdp->wd_worker_number, - status, - strerror(errno)); - } - -} // End of xdd_e2e_set_socket_opts() -/*----------------------------------------------------------------------*/ -/* - * xdd_e2e_prt_socket_opts() - print the currnet options for specified - * socket. - * - */ -void -xdd_e2e_prt_socket_opts(int skt) { - int level = SOL_SOCKET; - int sockbuf_sizs; - int sockbuf_sizr; - int reuse_addr; - socklen_t optlen; - - - optlen = sizeof(sockbuf_sizs); - getsockopt(skt,level,SO_SNDBUF,(char *)&sockbuf_sizs,&optlen); - optlen = sizeof(sockbuf_sizr); - getsockopt(skt,level,SO_RCVBUF,(char *)&sockbuf_sizr,&optlen); - optlen = sizeof(reuse_addr); - getsockopt(skt,level,SO_REUSEADDR,(char *)&reuse_addr,&optlen); -} // End of xdd_e2e_prt_socket_opts() - -/*----------------------------------------------------------------------*/ -/* xdd_e2e_err(fmt...) - * - * General-purpose error handling routine. Prints the short message - * provided to standard error, along with some boilerplate information - * such as the program name and errno value. Any remaining arguments - * are used in printed message (the usage here takes the same form as - * printf()). - */ -void -xdd_e2e_err(worker_data_t *wdp, char const *whence, char const *fmt, ...) { -#ifdef WIN32 - LPVOID lpMsgBuf; - fprintf(xgp->errout, "last error was %d\n", WSAGetLastError()); - FormatMessage( - FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - WSAGetLastError(), - MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language - (LPTSTR) &lpMsgBuf, - 0, - NULL); - fprintf(xgp->errout,"Reason: %s",lpMsgBuf); -#endif /* WIN32 */ - fprintf(xgp->errout,"\n%s: %s: Target %d Worker Thread %d: ", - xgp->progname, - whence, - wdp->wd_tdp->td_target_number, - wdp->wd_worker_number); - fprintf(xgp->errout, "%s", fmt); - perror(" Reason"); - return; -} /* end of xdd_e2e_err() */ - -/*----------------------------------------------------------------------*/ -/* xdd_sockets_init() - Windows Only - * - * Windows requires the WinSock startup routine to be run - * before running a bunch of socket routines. We encapsulate - * that here in case some other environment needs something similar. - * - * Return values: 0 is good - init successful, -1 is bad - * - * The sample code I based this on happened to be requesting - * (and verifying) a WinSock DLL version 2.2 environment was - * present, and it worked, so I kept it that way. - */ -int32_t xdd_sockets_init(void) { -#ifdef WIN32 - WSADATA wsaData; /* Data structure used by WSAStartup */ - int wsastatus; /* status returned by WSAStartup */ - char *reason; - wsastatus = WSAStartup(MAKEWORD(2, 2), &wsaData); - if (wsastatus != 0) { /* Error in starting the network */ - switch (wsastatus) { - case WSASYSNOTREADY: - reason = "Network is down"; - break; - case WSAVERNOTSUPPORTED: - reason = "Request version of sockets <2.2> is not supported"; - break; - case WSAEINPROGRESS: - reason = "Another Windows Sockets operation is in progress"; - break; - case WSAEPROCLIM: - reason = "The limit of the number of sockets tasks has been exceeded"; - break; - case WSAEFAULT: - reason = "Program error: pointer to wsaData is not valid"; - break; - default: - reason = "Unknown error code"; - break; - }; - fprintf(xgp->errout,"%s: Error initializing network connection\nReason: %s\n", - xgp->progname, reason); - fflush(xgp->errout); - WSACleanup(); - return(-1); - } -#endif - - return(0); - -} /* end of xdd_sockets_init() */ - /*----------------------------------------------------------------------------*/ /* xdd_get_e2ep() - return a pointer to the xdd_e2e Data Structure */ diff --git a/src/xnet/xnet_end_to_end.c b/src/xnet/xnet_end_to_end.c index a2d2dc2f..2c124ed5 100644 --- a/src/xnet/xnet_end_to_end.c +++ b/src/xnet/xnet_end_to_end.c @@ -146,7 +146,7 @@ int32_t xint_e2e_xni_send(worker_data_t *wdp) { tdp = wdp->wd_tdp; e2ep = wdp->wd_e2ep; - de2eprintf("DEBUG_E2E: %lld: xdd_e2e_src_send: Target: %d: Worker: %d: ENTER: e2ep=%p: e2e_datap=%p\n",(long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, e2ep, e2ep->e2e_datap); + de2eprintf("DEBUG_E2E: %lld: xdd_e2e_src_send: Target: %d: Worker: %d: ENTER: e2ep=%p\n",(long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, e2ep); /* Some timestamp code */ if (tdp->td_ts_table.ts_options & (TS_ON | TS_TRIGGERED)) { @@ -161,9 +161,8 @@ int32_t xint_e2e_xni_send(worker_data_t *wdp) { wdp->wd_e2ep->xni_wd_buf); xni_target_buffer_set_data_length(wdp->wd_task.task_xfer_size, wdp->wd_e2ep->xni_wd_buf); - e2ep->e2e_xfer_size = wdp->wd_task.task_xfer_size; - de2eprintf("DEBUG_E2E: %lld: xdd_e2e_src_send: Target: %d: Worker: %d: Preparing to send %d bytes: e2ep=%p: e2e_datap=%p: e2e_xfer_size=%d: e2eh_data_length=%lld\n",(long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, e2ep->e2e_xfer_size,e2ep,e2ep->e2e_datap,e2ep->e2e_xfer_size,(long long int)xni_target_buffer_data_length(wdp->wd_e2ep->xni_wd_buf)); + de2eprintf("DEBUG_E2E: %lld: xdd_e2e_src_send: Target: %d: Worker: %d: Preparing to send: e2ep=%p: e2eh_data_length=%lld\n",(long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number,e2ep,(long long int)xni_target_buffer_data_length(wdp->wd_e2ep->xni_wd_buf)); nclk_now(&wdp->wd_counters.tc_current_net_start_time); @@ -174,14 +173,13 @@ int32_t xint_e2e_xni_send(worker_data_t *wdp) { // Keep a pointer to the data portion of the buffer wdp->wd_task.task_datap = xni_target_buffer_data(wdp->wd_e2ep->xni_wd_buf); - wdp->wd_e2ep->e2e_datap = wdp->wd_task.task_datap; nclk_now(&wdp->wd_counters.tc_current_net_end_time); // Time stamp if requested if (tdp->td_ts_table.ts_options & (TS_ON | TS_TRIGGERED)) { ttep = &tdp->td_ts_table.ts_hdrp->tsh_tte[wdp->wd_ts_entry]; - ttep->tte_net_xfer_size = e2ep->e2e_xfer_size; + ttep->tte_net_xfer_size = xni_target_buffer_data_length(e2ep->xni_wd_buf); ttep->tte_net_start = wdp->wd_counters.tc_current_net_start_time; ttep->tte_net_end = wdp->wd_counters.tc_current_net_end_time; ttep->tte_net_processor_end = xdd_get_processor(); @@ -191,11 +189,6 @@ int32_t xint_e2e_xni_send(worker_data_t *wdp) { // Calculate the Send/Receive time by the time it took the last sendto() to run e2ep->e2e_sr_time = (wdp->wd_counters.tc_current_net_end_time - wdp->wd_counters.tc_current_net_start_time); - //if (bytes_sent != e2ep->e2e_xfer_size) { - // xdd_e2e_err(wdp,"xdd_e2e_src_send","ERROR: could not send header+data from e2e source\n"); - // return(-1); - //} - de2eprintf("DEBUG_E2E: %lld: xdd_e2e_src_send: Target: %d: Worker: %d: EXIT...\n",(long long int)pclk_now(),tdp->td_target_number, wdp->wd_worker_number); return(0); @@ -231,21 +224,15 @@ int32_t xint_e2e_xni_recv(worker_data_t *wdp) { if (XNI_OK == status) { /* Assemble pointers into the worker's target buffer */ wdp->wd_task.task_datap = xni_target_buffer_data(wdp->wd_e2ep->xni_wd_buf); - wdp->wd_e2ep->e2e_datap = wdp->wd_task.task_datap; } else if (XNI_EOF == status) { wdp->wd_task.task_datap = NULL; - wdp->wd_e2ep->e2e_datap = NULL; wdp->wd_e2ep->received_eof = TRUE; } else { fprintf(xgp->errout, "Error receiving data via XNI."); return -1; } - //de2eprintf("DEBUG_E2E: %lld: xdd_e2e_dest_recv: Target: %d: Worker: %d: Preparing to send %d bytes: e2ep=%p: e2ehp=%p: e2e_datap=%p: e2e_xfer_size=%d: e2eh_data_length=%lld\n",(long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number, e2ep->e2e_xfer_size,e2ep,e2ehp,e2ep->e2e_datap,e2ep->e2e_xfer_size,(long long int)e2ehp->e2eh_data_length); - //xgp->global_options |= GO_DEBUG_E2E; - //if (xgp->global_options & GO_DEBUG_E2E) xdd_show_e2e_header(wdp->wd_e2ep->e2e_hdrp); - /* Local aliases */ e2ep = wdp->wd_e2ep; @@ -262,7 +249,7 @@ int32_t xint_e2e_xni_recv(worker_data_t *wdp) { ttep->tte_net_start = wdp->wd_counters.tc_current_net_start_time; ttep->tte_net_end = wdp->wd_counters.tc_current_net_end_time; ttep->tte_net_processor_end = xdd_get_processor(); - ttep->tte_net_xfer_size = e2ep->e2e_recv_status; + ttep->tte_net_xfer_size = xni_target_buffer_data_length(e2ep->xni_wd_buf); ttep->tte_byte_offset = xni_target_buffer_target_offset(e2ep->xni_wd_buf); ttep->tte_disk_xfer_size = xni_target_buffer_data_length(e2ep->xni_wd_buf); ttep->tte_op_number = xni_target_buffer_sequence_number(e2ep->xni_wd_buf); From 99d5f9440a46c32492efe874ed81069ac5350ac0 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Thu, 17 Jul 2014 00:10:47 -0400 Subject: [PATCH 28/50] Remove dependency on E2E header --- src/tools/utils/matchadd_kernel_events.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/utils/matchadd_kernel_events.c b/src/tools/utils/matchadd_kernel_events.c index 98d718fe..98256a11 100644 --- a/src/tools/utils/matchadd_kernel_events.c +++ b/src/tools/utils/matchadd_kernel_events.c @@ -281,7 +281,7 @@ matchadd_kernel_events(int issource, int nthreads, int thread_id[], char *filesp xdd_data->tsh_tte[i].tte_net_start_k = ts_beg_op; xdd_data->tsh_tte[i].tte_net_end_k = ts_end_op; /* xdd eof stuff */ - if ( xdd_data->tsh_tte[i].tte_net_xfer_size != size_op && xdd_data->tsh_tte[i].tte_net_xfer_size != sizeof(xdd_e2e_header_t)) + if ( xdd_data->tsh_tte[i].tte_net_xfer_size != size_op ) fprintf(stderr, "xddop# %zd pid %d op %d size %d != %"PRId64"\n",i,thread_id[k],xdd_data->tsh_tte[i].tte_op_type,xdd_data->tsh_tte[i].tte_net_xfer_size,size_op); if ( xdd_data->tsh_tte[i].tte_net_xfer_calls != nops_op && nops_op > 0) fprintf(stderr, "xddop# %zd pid %d nops %d != %"PRId64"\n",i,thread_id[k],xdd_data->tsh_tte[i].tte_net_xfer_calls,nops_op); From 6d0eb4ef5bfad5e9b94336725cf2b9e376ce2507 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Thu, 17 Jul 2014 01:44:28 -0400 Subject: [PATCH 29/50] Move E2E to to xnet/ --- src/base/target_init.c | 2 +- src/base/worker_thread.c | 4 +- src/base/worker_thread_init.c | 2 +- src/client/parse.c | 2 +- src/client/parse_func.c | 8 +- src/common/target_data.c | 2 +- src/common/xint_prototypes.h | 16 +-- src/net/end_to_end.c | 53 --------- src/net/end_to_end_init.c | 201 -------------------------------- src/net/module.mk | 4 +- src/public/libxdd.c | 2 +- src/xnet/xnet_end_to_end.c | 25 ++++ src/xnet/xnet_end_to_end_init.c | 176 +++++++++++++++++++++++++++- 13 files changed, 217 insertions(+), 280 deletions(-) delete mode 100644 src/net/end_to_end.c delete mode 100644 src/net/end_to_end_init.c diff --git a/src/base/target_init.c b/src/base/target_init.c index fdcfa907..8a425241 100644 --- a/src/base/target_init.c +++ b/src/base/target_init.c @@ -148,7 +148,7 @@ xint_target_init(target_data_t *tdp) { // Special setup for an End-to-End operation if (xint_is_e2e(tdp)) { - status = xdd_e2e_target_init(tdp); + status = xint_e2e_target_init(tdp); if (status) return(-1); } diff --git a/src/base/worker_thread.c b/src/base/worker_thread.c index 440511da..9d478c80 100644 --- a/src/base/worker_thread.c +++ b/src/base/worker_thread.c @@ -76,8 +76,8 @@ xdd_worker_thread(void *pin) { xdd_worker_thread_cleanup(wdp); return(0); case TASK_REQ_EOF: - // E2E Source Side only - send EOF packets to Destination - status = xdd_e2e_eof_source_side(wdp); + // E2E Source Side only + status = xint_e2e_eof_source_side(wdp); if (status) // Only set the status in the Target Data Struct if it is non-zero tdp->td_counters.tc_current_io_status = status; break; diff --git a/src/base/worker_thread_init.c b/src/base/worker_thread_init.c index 5f1c66a1..52a09c8a 100644 --- a/src/base/worker_thread_init.c +++ b/src/base/worker_thread_init.c @@ -172,7 +172,7 @@ xdd_worker_thread_init(worker_data_t *wdp) { // Set up for an End-to-End operation (if requested) if (tdp->td_target_options & TO_ENDTOEND) { if (tdp->td_target_options & (TO_E2E_DESTINATION|TO_E2E_SOURCE)) { - status = xdd_e2e_worker_init(wdp); + status = xint_e2e_worker_init(wdp); } else { // Not sure which side of the E2E this target is supposed to be.... fprintf(xgp->errout,"%s: xdd_worker_thread_init: Target %d WorkerThread %d: Cannot determine which side of the E2E operation this target is supposed to be.\n", xgp->progname, diff --git a/src/client/parse.c b/src/client/parse.c index a8b706a3..9031f775 100644 --- a/src/client/parse.c +++ b/src/client/parse.c @@ -388,7 +388,7 @@ xdd_get_target_datap(xdd_plan_t *planp, int32_t target_number, char *op) { if (planp->plan_options & PLAN_ENDTOEND) { if (NULL == tdp->td_e2ep) { // If there is no e2e struct then allocate one. - tdp->td_e2ep = xdd_get_e2ep(); + tdp->td_e2ep = xint_get_e2ep(); if (NULL == tdp->td_e2ep) { fprintf(xgp->errout,"%s: ERROR: Cannot allocate %d bytes of memory for Target Data Struct END TO END Data Structure for target %d\n", xgp->progname, (int)sizeof(xint_data_pattern_t), target_number); diff --git a/src/client/parse_func.c b/src/client/parse_func.c index 35a05c44..0f2d181b 100644 --- a/src/client/parse_func.c +++ b/src/client/parse_func.c @@ -840,7 +840,7 @@ xddfunc_endtoend(xdd_plan_t *planp, int32_t argc, char *argv[], uint32_t flags) tdp = xdd_get_target_datap(planp, target_number, argv[0]); if (tdp == NULL) return(-1); if (NULL == tdp->td_e2ep) { // If there is no e2e struct then allocate one. - tdp->td_e2ep = xdd_get_e2ep(); + tdp->td_e2ep = xint_get_e2ep(); if (NULL == tdp->td_e2ep) { fprintf(xgp->errout,"%s: ERROR: Cannot allocate %d bytes of memory for Target Data Struct END TO END Data Structure for target %d\n", xgp->progname, (int)sizeof(xint_data_pattern_t), target_number); @@ -852,7 +852,7 @@ xddfunc_endtoend(xdd_plan_t *planp, int32_t argc, char *argv[], uint32_t flags) i = 0; while (tdp) { if (NULL == tdp->td_e2ep) { // If there is no e2e struct then allocate one. - tdp->td_e2ep = xdd_get_e2ep(); + tdp->td_e2ep = xint_get_e2ep(); if (NULL == tdp->td_e2ep) { fprintf(xgp->errout,"%s: ERROR: Cannot allocate %d bytes of memory for Target Data Struct END TO END Data Structure for target %d\n", xgp->progname, (int)sizeof(xint_data_pattern_t), target_number); @@ -3166,7 +3166,7 @@ xddfunc_restart(xdd_plan_t *planp, int32_t argc, char *argv[], uint32_t flags) if (rp == NULL) return(-1); if (NULL == tdp->td_e2ep) // If there is no e2e struct, then allocate one - tdp->td_e2ep = xdd_get_e2ep(); + tdp->td_e2ep = xint_get_e2ep(); if (tdp->td_e2ep == NULL) // If there is still no e2e struct then return -1 return(-1); @@ -3188,7 +3188,7 @@ xddfunc_restart(xdd_plan_t *planp, int32_t argc, char *argv[], uint32_t flags) if (rp == NULL) return(-1); if (NULL == tdp->td_e2ep) // If there is no e2e struct, then allocate one - tdp->td_e2ep = xdd_get_e2ep(); + tdp->td_e2ep = xint_get_e2ep(); if (tdp->td_e2ep == NULL) // If there is still no e2e struct then return -1 return(-1); diff --git a/src/common/target_data.c b/src/common/target_data.c index f3551fa0..4d5b9cc9 100644 --- a/src/common/target_data.c +++ b/src/common/target_data.c @@ -213,7 +213,7 @@ xdd_create_worker_data(target_data_t *tdp, int32_t q) { // Allocate and initialize the End-to-End structure if needed if (tdp->td_target_options & TO_ENDTOEND) { - wdp->wd_e2ep = xdd_get_e2ep(); + wdp->wd_e2ep = xint_get_e2ep(); if (NULL == wdp->wd_e2ep) { fprintf(xgp->errout,"%s: ERROR: Cannot allocate %d bytes of memory for WORKER_DATA END TO END Data Structure for worker %d\n", xgp->progname, (int)sizeof(xint_data_pattern_t), q); diff --git a/src/common/xint_prototypes.h b/src/common/xint_prototypes.h index 86d4fa76..72d8131d 100644 --- a/src/common/xint_prototypes.h +++ b/src/common/xint_prototypes.h @@ -50,15 +50,6 @@ void xdd_show_ts_table(xint_timestamp_t *ts_tablep, int target_number); void xdd_show_ts_header(xdd_ts_header_t *ts_hdrp, int target_number); void xdd_show_results_data(results_t *rp, char *dumptype, xdd_plan_t *planp); -// end_to_end.c -int32_t xdd_e2e_eof_source_side(worker_data_t *wdp); - -// end_to_end_init.c -int32_t xdd_e2e_target_init(target_data_t *tdp); -int32_t xdd_e2e_worker_init(worker_data_t *wdp); -int32_t xdd_e2e_src_init(worker_data_t *wdp); -int32_t xdd_e2e_dest_init(worker_data_t *wdp); - // global_clock.c in_addr_t xdd_init_global_clock_network(char *hostname); void xdd_init_global_clock(nclk_t *nclkp); @@ -142,7 +133,6 @@ int xdd_parse_target_number(xdd_plan_t* planp, int32_t argc, char *argv[], target_data_t *xdd_get_target_datap(xdd_plan_t* planp, int32_t target_number, char *op); xint_restart_t *xdd_get_restartp(target_data_t *tdp); xint_raw_t *xdd_get_rawp(target_data_t *tdp); -xint_e2e_t *xdd_get_e2ep(void); xint_throttle_t *xdd_get_throtp(target_data_t *tdp); xint_triggers_t *xdd_get_trigp(target_data_t *tdp); xint_extended_stats_t *xdd_get_esp(target_data_t *tdp); @@ -387,12 +377,18 @@ void xdd_start_restart_monitor(xdd_plan_t *planp); void xdd_start_interactive(xdd_plan_t *planp); // xnet_end_to_end_init.c +int32_t xint_e2e_target_init(target_data_t *tdp); int32_t xint_e2e_xni_init(target_data_t *tdp); +int32_t xint_e2e_worker_init(worker_data_t *wdp); +int32_t xint_e2e_src_init(worker_data_t *wdp); +int32_t xint_e2e_dest_init(worker_data_t *wdp); +xint_e2e_t *xint_get_e2ep(void); // xnet_end_to_end.c int32_t xint_e2e_dest_connect(target_data_t *tdp); int32_t xint_e2e_src_connect(target_data_t *tdp); int32_t xint_e2e_xni_send(worker_data_t *wdp); +int32_t xint_e2e_eof_source_side(worker_data_t *wdp); int32_t xint_e2e_xni_recv(worker_data_t *wdp); int xint_is_e2e(const target_data_t *tdp); #endif diff --git a/src/net/end_to_end.c b/src/net/end_to_end.c deleted file mode 100644 index cbdfef94..00000000 --- a/src/net/end_to_end.c +++ /dev/null @@ -1,53 +0,0 @@ -/* - * XDD - a data movement and benchmarking toolkit - * - * Copyright (C) 1992-2013 I/O Performance, Inc. - * Copyright (C) 2009-2013 UT-Battelle, LLC - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License version 2, as published by the Free Software - * Foundation. See file COPYING. - * - */ -/* - * This file contains the subroutines necessary the end-to-end - * Send and Receive functions. - */ -#include "xint.h" - -/*----------------------------------------------------------------------*/ -/* xdd_e2e_eof_source_side() - End-Of-File processing for Source - * Return values: 0 is good, -1 is bad - */ -int32_t -xdd_e2e_eof_source_side(worker_data_t *wdp) { - target_data_t *tdp; - xint_e2e_t *e2ep; // Pointer to the E2E struct for this worker - - tdp = wdp->wd_tdp; - e2ep = wdp->wd_e2ep; - -if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_eof_source_side: Target %d Worker: %d: ENTER: \n", (long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number); - - /* If this is XNI, just short circuit */ - if (PLAN_ENABLE_XNI & tdp->td_planp->plan_options) { - e2ep->e2e_send_status = 0; - e2ep->e2e_sr_time = 0; - return 0; - } - - // we only support XNI now - return -1; -} /* end of xdd_e2e_eof_source_side() */ - -/* - * Local variables: - * indent-tabs-mode: t - * default-tab-width: 4 - * c-indent-level: 4 - * c-basic-offset: 4 - * End: - * - * vim: ts=4 sts=4 sw=4 noexpandtab - */ diff --git a/src/net/end_to_end_init.c b/src/net/end_to_end_init.c deleted file mode 100644 index 1192af22..00000000 --- a/src/net/end_to_end_init.c +++ /dev/null @@ -1,201 +0,0 @@ -/* - * XDD - a data movement and benchmarking toolkit - * - * Copyright (C) 1992-2013 I/O Performance, Inc. - * Copyright (C) 2009-2013 UT-Battelle, LLC - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License version 2, as published by the Free Software - * Foundation. See file COPYING. - * - */ -/* - * This file contains the subroutines necessary to perform initialization - * for the end-to-end option. - */ -#include "xint.h" - -/*----------------------------------------------------------------------*/ -/* xdd_e2e_target_init() - init socket library - * This routine is called during target initialization to initialize the - * socket library. - * - * Return values: 0 is good, -1 is bad - * - */ -int32_t -xdd_e2e_target_init(target_data_t *tdp) { - xint_restart_t *rp; // pointer to a restart structure - int status; - - // Perform XNI initialization if required - status = xint_e2e_xni_init(tdp); - if (status == -1) { - fprintf(xgp->errout,"%s: xdd_e2e_target_init: could not initialize XNI for e2e target\n",xgp->progname); - return(-1); - } - - // Restart processing if necessary - if ((tdp->td_target_options & TO_RESTART_ENABLE) && (tdp->td_restartp)) { // Check to see if restart was requested - // Set the last_committed_byte_offset to 0 - rp = tdp->td_restartp; - rp->last_committed_byte_offset = rp->byte_offset; - rp->last_committed_length = 0; - } - - return(0); -} - -/*----------------------------------------------------------------------*/ -/* xdd_e2e_worker_init() - init source and destination sides - * This routine is called during Worker Thread initialization to initialize - * a source or destination Worker Thread. - * - * Return values: 0 is good, -1 is bad - * - */ -int32_t -xdd_e2e_worker_init(worker_data_t *wdp) { - target_data_t *tdp; - int status; - in_addr_t addr; - - tdp = wdp->wd_tdp; - wdp->wd_e2ep->e2e_sr_time = 0; - - if(wdp->wd_e2ep->e2e_dest_hostname == NULL) { - fprintf(xgp->errout,"%s: xdd_e2e_worker_init: Target %d Worker Thread %d: No DESTINATION host name or IP address specified for this end-to-end operation.\n", - xgp->progname, - tdp->td_target_number, - wdp->wd_worker_number); - fprintf(xgp->errout,"%s: xdd_e2e_worker_init: Target %d Worker Thread %d: Use the '-e2e destination' option to specify the DESTINATION host name or IP address.\n", - xgp->progname, - tdp->td_target_number, - wdp->wd_worker_number); - return(-1); - } - - // Get the IP address of the destination host - status = xint_lookup_addr(wdp->wd_e2ep->e2e_dest_hostname, 0, &addr); - if (status) { - fprintf(xgp->errout, "%s: xdd_e2e_worker_init: unable to identify host '%s'\n", - xgp->progname, wdp->wd_e2ep->e2e_dest_hostname); - return(-1); - } - - // Convert to host byte order - wdp->wd_e2ep->e2e_dest_addr = ntohl(addr); - - if (tdp->td_target_options & TO_E2E_DESTINATION) { // This is the Destination side of an End-to-End - status = xdd_e2e_dest_init(wdp); - } else if (tdp->td_target_options & TO_E2E_SOURCE) { // This is the Source side of an End-to-End - status = xdd_e2e_src_init(wdp); - } else { // Should never reach this point - status = -1; - } - - return status; -} // xdd_e2e_worker_init() - -/*----------------------------------------------------------------------*/ -/* xdd_e2e_src_init() - init the source side - * This routine is called from the xdd io thread before all the action - * starts. When calling this routine, it is because this thread is on the - * "source" side of an End-to-End test. Hence, this routine needs - * to set up a socket on the appropriate port - * - * Return values: 0 is good, -1 is bad - * - */ -int32_t -xdd_e2e_src_init(worker_data_t *wdp) { - target_data_t *tdp; - xint_e2e_t *e2ep; // Pointer to the E2E data struct - - - tdp = wdp->wd_tdp; - e2ep = wdp->wd_e2ep; - - // Check to make sure that the source target is actually *reading* the data from a file or device - if (tdp->td_rwratio < 1.0) { // Something is wrong - the source file/device is not 100% read - fprintf(xgp->errout,"%s: xdd_e2e_src_init: Target %d Worker Thread %d: Error - E2E source file '%s' is not being *read*: rwratio=%5.2f is not valid\n", - xgp->progname, - tdp->td_target_number, - wdp->wd_worker_number, - tdp->td_target_full_pathname, - tdp->td_rwratio); - return(-1); - } - - // Init the relevant variables - e2ep->e2e_msg_sequence_number = 0; - - return(0); - -} /* end of xdd_e2e_src_init() */ - -/*----------------------------------------------------------------------*/ -/* xdd_e2e_dest_init() - init the destination side - * This routine is called by a Worker Thread on the "destination" side of an - * end_to_end operation and is passed a pointer to the Data Struct of the - * requesting Worker Thread. - * - * Return values: 0 is good, -1 is bad - * - */ -int32_t -xdd_e2e_dest_init(worker_data_t *wdp) { - target_data_t *tdp; - - - tdp = wdp->wd_tdp; - // Check to make sure that the destination target is actually *writing* the data it receives to a file or device - if (tdp->td_rwratio > 0.0) { // Something is wrong - the destination file/device is not 100% write - fprintf(xgp->errout,"%s: xdd_e2e_dest_init: Target %d Worker Thread %d: Error - E2E destination file/device '%s' is not being *written*: rwratio=%5.2f is not valid\n", - xgp->progname, - tdp->td_target_number, - wdp->wd_worker_number, - tdp->td_target_full_pathname, - tdp->td_rwratio); - return(-1); - } - - // Initialize the message counter and sequencer to 0 - wdp->wd_e2ep->e2e_msg_sequence_number = 0; - - // Clear the end-of-file flag - wdp->wd_e2ep->received_eof = FALSE; - - return(0); - -} /* end of xdd_e2e_dest_init() */ - -/*----------------------------------------------------------------------------*/ -/* xdd_get_e2ep() - return a pointer to the xdd_e2e Data Structure - */ -xint_e2e_t * -xdd_get_e2ep(void) { - xint_e2e_t *e2ep; - - e2ep = malloc(sizeof(xint_e2e_t)); - if (e2ep == NULL) { - fprintf(xgp->errout,"%s: ERROR: Cannot allocate %d bytes of memory for E2E data structure \n", - xgp->progname, (int)sizeof(xint_e2e_t)); - return(NULL); - } - memset(e2ep, 0, sizeof(xint_e2e_t)); - - return(e2ep); -} /* End of xdd_get_e2ep() */ - -/* - * Local variables: - * indent-tabs-mode: t - * default-tab-width: 4 - * c-indent-level: 4 - * c-basic-offset: 4 - * End: - * - * vim: ts=4 sts=4 sw=4 noexpandtab - */ diff --git a/src/net/module.mk b/src/net/module.mk index 00e3d0cf..d2b685a5 100644 --- a/src/net/module.mk +++ b/src/net/module.mk @@ -3,7 +3,5 @@ # DIR := src/net -NET_SRC := $(DIR)/end_to_end.c \ - $(DIR)/end_to_end_init.c \ - $(DIR)/read_after_write.c \ +NET_SRC := $(DIR)/read_after_write.c \ $(DIR)/net_utils.c diff --git a/src/public/libxdd.c b/src/public/libxdd.c index 06ad1027..cb392b9c 100644 --- a/src/public/libxdd.c +++ b/src/public/libxdd.c @@ -240,7 +240,7 @@ int local_target_init(target_data_t *tdp, // Setup a data pattern and e2e buffer before initialization tdp->td_dpp = malloc(sizeof(*tdp->td_dpp)); xdd_data_pattern_init(tdp->td_dpp); - tdp->td_e2ep = xdd_get_e2ep(); + tdp->td_e2ep = xint_get_e2ep(); // Now initialize the target data xdd_init_new_target_data(tdp, target_idx); diff --git a/src/xnet/xnet_end_to_end.c b/src/xnet/xnet_end_to_end.c index 2c124ed5..6e8104d7 100644 --- a/src/xnet/xnet_end_to_end.c +++ b/src/xnet/xnet_end_to_end.c @@ -195,6 +195,31 @@ int32_t xint_e2e_xni_send(worker_data_t *wdp) { } /* end of xdd_e2e_src_send() */ +/*----------------------------------------------------------------------*/ +/* xint_e2e_eof_source_side() - End-Of-File processing for Source + * Return values: 0 is good, -1 is bad + */ +int32_t +xint_e2e_eof_source_side(worker_data_t *wdp) { + target_data_t *tdp; + xint_e2e_t *e2ep; // Pointer to the E2E struct for this worker + + tdp = wdp->wd_tdp; + e2ep = wdp->wd_e2ep; + +if (xgp->global_options & GO_DEBUG_E2E) fprintf(stderr,"DEBUG_E2E: %lld: xdd_e2e_eof_source_side: Target %d Worker: %d: ENTER: \n", (long long int)pclk_now(), tdp->td_target_number, wdp->wd_worker_number); + + /* If this is XNI, just short circuit */ + if (PLAN_ENABLE_XNI & tdp->td_planp->plan_options) { + e2ep->e2e_send_status = 0; + e2ep->e2e_sr_time = 0; + return 0; + } + + // we only support XNI now + return -1; +} /* end of xdd_e2e_eof_source_side() */ + /* * xint_e2e_xni_recv() - recv the data from source at destination * diff --git a/src/xnet/xnet_end_to_end_init.c b/src/xnet/xnet_end_to_end_init.c index 19d2388f..bb3e48d8 100644 --- a/src/xnet/xnet_end_to_end_init.c +++ b/src/xnet/xnet_end_to_end_init.c @@ -37,9 +37,40 @@ /*----------------------------------------------------------------------*/ -/* xdd_e2e_target_init() - init socket library +/* xint_e2e_target_init() - init target structure for E2E * This routine is called during target initialization to initialize the - * socket library. + * target data structure for end-to-end. + * + * Return values: 0 is good, -1 is bad + * + */ +int32_t +xint_e2e_target_init(target_data_t *tdp) { + xint_restart_t *rp; // pointer to a restart structure + int status; + + // Perform XNI initialization if required + status = xint_e2e_xni_init(tdp); + if (status == -1) { + fprintf(xgp->errout,"%s: xint_e2e_target_init: could not initialize XNI for e2e target\n",xgp->progname); + return(-1); + } + + // Restart processing if necessary + if ((tdp->td_target_options & TO_RESTART_ENABLE) && (tdp->td_restartp)) { // Check to see if restart was requested + // Set the last_committed_byte_offset to 0 + rp = tdp->td_restartp; + rp->last_committed_byte_offset = rp->byte_offset; + rp->last_committed_length = 0; + } + + return(0); +} + +/*----------------------------------------------------------------------*/ +/* xint_e2e_xni_init() - init network library + * This routine is called during target initialization to initialize the + * network library. * * Return values: 0 is good, -1 is bad * @@ -69,6 +100,147 @@ int32_t xint_e2e_xni_init(target_data_t *tdp) { return(0); } +/*----------------------------------------------------------------------*/ +/* xint_e2e_worker_init() - init source and destination sides + * This routine is called during Worker Thread initialization to initialize + * a source or destination Worker Thread. + * + * Return values: 0 is good, -1 is bad + * + */ +int32_t +xint_e2e_worker_init(worker_data_t *wdp) { + target_data_t *tdp; + int status; + in_addr_t addr; + + tdp = wdp->wd_tdp; + wdp->wd_e2ep->e2e_sr_time = 0; + + if(wdp->wd_e2ep->e2e_dest_hostname == NULL) { + fprintf(xgp->errout,"%s: xint_e2e_worker_init: Target %d Worker Thread %d: No DESTINATION host name or IP address specified for this end-to-end operation.\n", + xgp->progname, + tdp->td_target_number, + wdp->wd_worker_number); + fprintf(xgp->errout,"%s: xint_e2e_worker_init: Target %d Worker Thread %d: Use the '-e2e destination' option to specify the DESTINATION host name or IP address.\n", + xgp->progname, + tdp->td_target_number, + wdp->wd_worker_number); + return(-1); + } + + // Get the IP address of the destination host + status = xint_lookup_addr(wdp->wd_e2ep->e2e_dest_hostname, 0, &addr); + if (status) { + fprintf(xgp->errout, "%s: xint_e2e_worker_init: unable to identify host '%s'\n", + xgp->progname, wdp->wd_e2ep->e2e_dest_hostname); + return(-1); + } + + // Convert to host byte order + wdp->wd_e2ep->e2e_dest_addr = ntohl(addr); + + if (tdp->td_target_options & TO_E2E_DESTINATION) { // This is the Destination side of an End-to-End + status = xint_e2e_dest_init(wdp); + } else if (tdp->td_target_options & TO_E2E_SOURCE) { // This is the Source side of an End-to-End + status = xint_e2e_src_init(wdp); + } else { // Should never reach this point + status = -1; + } + + return status; +} // xint_e2e_worker_init() + +/*----------------------------------------------------------------------*/ +/* xint_e2e_src_init() - init the source side + * This routine is called from the xdd io thread before all the action + * starts. When calling this routine, it is because this thread is on the + * "source" side of an End-to-End transfer. + * + * Return values: 0 is good, -1 is bad + * + */ +int32_t +xint_e2e_src_init(worker_data_t *wdp) { + target_data_t *tdp; + xint_e2e_t *e2ep; // Pointer to the E2E data struct + + + tdp = wdp->wd_tdp; + e2ep = wdp->wd_e2ep; + + // Check to make sure that the source target is actually *reading* the data from a file or device + if (tdp->td_rwratio < 1.0) { // Something is wrong - the source file/device is not 100% read + fprintf(xgp->errout,"%s: xint_e2e_src_init: Target %d Worker Thread %d: Error - E2E source file '%s' is not being *read*: rwratio=%5.2f is not valid\n", + xgp->progname, + tdp->td_target_number, + wdp->wd_worker_number, + tdp->td_target_full_pathname, + tdp->td_rwratio); + return(-1); + } + + // Init the relevant variables + e2ep->e2e_msg_sequence_number = 0; + + return(0); + +} /* end of xdd_e2e_src_init() */ + +/*----------------------------------------------------------------------*/ +/* xint_e2e_dest_init() - init the destination side + * This routine is called by a Worker Thread on the "destination" side of an + * end_to_end operation and is passed a pointer to the Data Struct of the + * requesting Worker Thread. + * + * Return values: 0 is good, -1 is bad + * + */ +int32_t +xint_e2e_dest_init(worker_data_t *wdp) { + target_data_t *tdp; + + + tdp = wdp->wd_tdp; + // Check to make sure that the destination target is actually *writing* the data it receives to a file or device + if (tdp->td_rwratio > 0.0) { // Something is wrong - the destination file/device is not 100% write + fprintf(xgp->errout,"%s: xdd_e2e_dest_init: Target %d Worker Thread %d: Error - E2E destination file/device '%s' is not being *written*: rwratio=%5.2f is not valid\n", + xgp->progname, + tdp->td_target_number, + wdp->wd_worker_number, + tdp->td_target_full_pathname, + tdp->td_rwratio); + return(-1); + } + + // Initialize the message counter and sequencer to 0 + wdp->wd_e2ep->e2e_msg_sequence_number = 0; + + // Clear the end-of-file flag + wdp->wd_e2ep->received_eof = FALSE; + + return(0); + +} /* end of xdd_e2e_dest_init() */ + +/*----------------------------------------------------------------------------*/ +/* xint_get_e2ep() - allocate a new E2E Data Structure + */ +xint_e2e_t * +xint_get_e2ep(void) { + xint_e2e_t *e2ep; + + e2ep = malloc(sizeof(xint_e2e_t)); + if (e2ep == NULL) { + fprintf(xgp->errout,"%s: ERROR: Cannot allocate %d bytes of memory for E2E data structure \n", + xgp->progname, (int)sizeof(xint_e2e_t)); + return(NULL); + } + memset(e2ep, 0, sizeof(xint_e2e_t)); + + return(e2ep); +} /* End of xint_get_e2ep() */ + /* * Local variables: * indent-tabs-mode: t From e32010e7f5ac60c3ff4d340c8f94920d74e1e80a Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Thu, 17 Jul 2014 18:09:16 -0400 Subject: [PATCH 30/50] Tidy up E2E init helper functions --- src/common/xint_prototypes.h | 3 -- src/xnet/xnet_end_to_end_init.c | 56 ++++++++++++--------------------- 2 files changed, 20 insertions(+), 39 deletions(-) diff --git a/src/common/xint_prototypes.h b/src/common/xint_prototypes.h index 72d8131d..63694273 100644 --- a/src/common/xint_prototypes.h +++ b/src/common/xint_prototypes.h @@ -378,10 +378,7 @@ void xdd_start_interactive(xdd_plan_t *planp); // xnet_end_to_end_init.c int32_t xint_e2e_target_init(target_data_t *tdp); -int32_t xint_e2e_xni_init(target_data_t *tdp); int32_t xint_e2e_worker_init(worker_data_t *wdp); -int32_t xint_e2e_src_init(worker_data_t *wdp); -int32_t xint_e2e_dest_init(worker_data_t *wdp); xint_e2e_t *xint_get_e2ep(void); // xnet_end_to_end.c diff --git a/src/xnet/xnet_end_to_end_init.c b/src/xnet/xnet_end_to_end_init.c index bb3e48d8..03bffb76 100644 --- a/src/xnet/xnet_end_to_end_init.c +++ b/src/xnet/xnet_end_to_end_init.c @@ -36,6 +36,12 @@ #include "xni.h" +// forward declarations +static int init_xni(target_data_t*); +static int init_src_worker(worker_data_t*); +static int init_dest_worker(worker_data_t*); + + /*----------------------------------------------------------------------*/ /* xint_e2e_target_init() - init target structure for E2E * This routine is called during target initialization to initialize the @@ -50,7 +56,7 @@ xint_e2e_target_init(target_data_t *tdp) { int status; // Perform XNI initialization if required - status = xint_e2e_xni_init(tdp); + status = init_xni(tdp); if (status == -1) { fprintf(xgp->errout,"%s: xint_e2e_target_init: could not initialize XNI for e2e target\n",xgp->progname); return(-1); @@ -67,15 +73,9 @@ xint_e2e_target_init(target_data_t *tdp) { return(0); } -/*----------------------------------------------------------------------*/ -/* xint_e2e_xni_init() - init network library - * This routine is called during target initialization to initialize the - * network library. - * - * Return values: 0 is good, -1 is bad - * - */ -int32_t xint_e2e_xni_init(target_data_t *tdp) { +static int +init_xni(target_data_t *tdp) +{ int rc = 0; /* Create the XNI control block */ size_t num_threads = tdp->td_planp->number_of_iothreads; @@ -141,9 +141,9 @@ xint_e2e_worker_init(worker_data_t *wdp) { wdp->wd_e2ep->e2e_dest_addr = ntohl(addr); if (tdp->td_target_options & TO_E2E_DESTINATION) { // This is the Destination side of an End-to-End - status = xint_e2e_dest_init(wdp); + status = init_dest_worker(wdp); } else if (tdp->td_target_options & TO_E2E_SOURCE) { // This is the Source side of an End-to-End - status = xint_e2e_src_init(wdp); + status = init_src_worker(wdp); } else { // Should never reach this point status = -1; } @@ -151,17 +151,9 @@ xint_e2e_worker_init(worker_data_t *wdp) { return status; } // xint_e2e_worker_init() -/*----------------------------------------------------------------------*/ -/* xint_e2e_src_init() - init the source side - * This routine is called from the xdd io thread before all the action - * starts. When calling this routine, it is because this thread is on the - * "source" side of an End-to-End transfer. - * - * Return values: 0 is good, -1 is bad - * - */ -int32_t -xint_e2e_src_init(worker_data_t *wdp) { +static int +init_src_worker(worker_data_t *wdp) +{ target_data_t *tdp; xint_e2e_t *e2ep; // Pointer to the E2E data struct @@ -185,19 +177,11 @@ xint_e2e_src_init(worker_data_t *wdp) { return(0); -} /* end of xdd_e2e_src_init() */ +} -/*----------------------------------------------------------------------*/ -/* xint_e2e_dest_init() - init the destination side - * This routine is called by a Worker Thread on the "destination" side of an - * end_to_end operation and is passed a pointer to the Data Struct of the - * requesting Worker Thread. - * - * Return values: 0 is good, -1 is bad - * - */ -int32_t -xint_e2e_dest_init(worker_data_t *wdp) { +static int +init_dest_worker(worker_data_t *wdp) +{ target_data_t *tdp; @@ -221,7 +205,7 @@ xint_e2e_dest_init(worker_data_t *wdp) { return(0); -} /* end of xdd_e2e_dest_init() */ +} /*----------------------------------------------------------------------------*/ /* xint_get_e2ep() - allocate a new E2E Data Structure From 39ccda4eb5d34db1bb8b83514aeb2ce666beff50 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Thu, 17 Jul 2014 19:24:30 -0400 Subject: [PATCH 31/50] Properly disconnect e2e transfers --- src/base/target_cleanup.c | 10 +++++----- src/common/xint_prototypes.h | 1 + src/xnet/xnet_end_to_end.c | 25 ++++++++++++++----------- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/base/target_cleanup.c b/src/base/target_cleanup.c index 877811f9..d8afec69 100644 --- a/src/base/target_cleanup.c +++ b/src/base/target_cleanup.c @@ -42,12 +42,12 @@ xdd_target_thread_cleanup(target_data_t *tdp) { #endif } - /* On e2e XNI, part of cleanup includes closing the source side */ - if ((TO_ENDTOEND & tdp->td_target_options) && - (PLAN_ENABLE_XNI & tdp->td_planp->plan_options)) { - xni_close_connection(&tdp->td_e2ep->xni_td_conn); - } + // Disconnect if this is an e2e transfer + if (xint_is_e2e(tdp)) { + xint_e2e_disconnect(tdp); + } + // Free the I/O buffers for (size_t i = 0; i < tdp->io_buffers_count; i++) { free(tdp->io_buffers[i]); } diff --git a/src/common/xint_prototypes.h b/src/common/xint_prototypes.h index 63694273..66819e89 100644 --- a/src/common/xint_prototypes.h +++ b/src/common/xint_prototypes.h @@ -384,6 +384,7 @@ xint_e2e_t *xint_get_e2ep(void); // xnet_end_to_end.c int32_t xint_e2e_dest_connect(target_data_t *tdp); int32_t xint_e2e_src_connect(target_data_t *tdp); +int32_t xint_e2e_disconnect(target_data_t *tdp); int32_t xint_e2e_xni_send(worker_data_t *wdp); int32_t xint_e2e_eof_source_side(worker_data_t *wdp); int32_t xint_e2e_xni_recv(worker_data_t *wdp); diff --git a/src/xnet/xnet_end_to_end.c b/src/xnet/xnet_end_to_end.c index 6e8104d7..31b6b968 100644 --- a/src/xnet/xnet_end_to_end.c +++ b/src/xnet/xnet_end_to_end.c @@ -76,13 +76,6 @@ int32_t xint_e2e_src_connect(target_data_t *tdp) { return rc; } -int32_t xint_e2e_src_disconnect(target_data_t *tdp) { - - /* Perform XNI disconnect */ - int rc = xni_close_connection(&tdp->td_e2ep->xni_td_conn); - return rc; -} - int32_t xint_e2e_dest_connect(target_data_t *tdp) { int rc = 0; @@ -114,11 +107,21 @@ int32_t xint_e2e_dest_connect(target_data_t *tdp) { return rc; } -int32_t xint_e2e_dest_disconnect(target_data_t *tdp) { - - /* Perform XNI disconnect */ +/* + * xint_e2e_disconnect() - close connections and free resources + * This function will close all connections associated with the given + * target. + * + * Returns 0 on success, -1 on failure + */ +int32_t +xint_e2e_disconnect(target_data_t *tdp) +{ + //TODO: handle errors int rc = xni_close_connection(&tdp->td_e2ep->xni_td_conn); - return rc; + assert(XNI_OK == rc); + + return 0; } /* From 84e807ca9e2dadad36dae701d241fee1aade7c6c Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Tue, 22 Jul 2014 14:44:19 -0400 Subject: [PATCH 32/50] Abort after network errors --- src/xnet/xnet_end_to_end.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/xnet/xnet_end_to_end.c b/src/xnet/xnet_end_to_end.c index 31b6b968..284de4d0 100644 --- a/src/xnet/xnet_end_to_end.c +++ b/src/xnet/xnet_end_to_end.c @@ -56,6 +56,7 @@ int32_t xint_e2e_src_connect(target_data_t *tdp) { /* Resolve name to an IP */ rc = xint_lookup_addr(tdp->td_e2ep->e2e_address_table[e2e_idx].hostname, 0, &tdp->td_e2ep->e2e_dest_addr); + assert(0 == rc); struct in_addr addr = { .s_addr = tdp->td_e2ep->e2e_dest_addr }; char* ip_string = inet_ntoa(addr); fprintf(xgp->errout, "Dest host: %s Connect IP: %s Port: %d\n", tdp->td_e2ep->e2e_address_table[e2e_idx].hostname, ip_string, tdp->td_e2ep->e2e_address_table[e2e_idx].base_port); @@ -73,6 +74,9 @@ int32_t xint_e2e_src_connect(target_data_t *tdp) { bufset.reserved = getpagesize(); rc = xni_connect(tdp->xni_ctx, &xep, &bufset, &tdp->td_e2ep->xni_td_conn); + // translate the error code + rc = (XNI_OK == rc) ? 0 : -1; + return rc; } @@ -88,6 +92,7 @@ int32_t xint_e2e_dest_connect(target_data_t *tdp) { /* Resolve name to an IP */ rc = xint_lookup_addr(tdp->td_e2ep->e2e_address_table[e2e_idx].hostname, 0, &tdp->td_e2ep->e2e_dest_addr); + assert(0 == rc); struct in_addr addr = { .s_addr = tdp->td_e2ep->e2e_dest_addr }; char* ip_string = inet_ntoa(addr); @@ -104,6 +109,9 @@ int32_t xint_e2e_dest_connect(target_data_t *tdp) { bufset.reserved = getpagesize(); rc = xni_accept_connection(tdp->xni_ctx, &xep, &bufset, &tdp->td_e2ep->xni_td_conn); + // translate the error code + rc = (XNI_OK == rc) ? 0 : -1; + return rc; } From a98a0abb8d37e6c86f5ab1dbe99f168c55b7a12a Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Tue, 22 Jul 2014 14:46:53 -0400 Subject: [PATCH 33/50] Keep multiple XNI connection objects Only one connection can be used currently, but multinic will require multiple connections. --- src/base/target_cleanup.c | 4 ++++ src/base/worker_thread_init.c | 3 ++- src/common/end_to_end.h | 3 ++- src/common/target_data.c | 2 ++ src/common/xint_prototypes.h | 1 + src/xnet/xnet_end_to_end.c | 41 ++++++++++++++++++++++++++++----- src/xnet/xnet_end_to_end_init.c | 9 ++++++++ 7 files changed, 55 insertions(+), 8 deletions(-) diff --git a/src/base/target_cleanup.c b/src/base/target_cleanup.c index d8afec69..2a59cfd7 100644 --- a/src/base/target_cleanup.c +++ b/src/base/target_cleanup.c @@ -45,6 +45,10 @@ xdd_target_thread_cleanup(target_data_t *tdp) { // Disconnect if this is an e2e transfer if (xint_is_e2e(tdp)) { xint_e2e_disconnect(tdp); + + // Free the connections + tdp->td_e2ep->xni_td_connections_count = 0; + tdp->td_e2ep->xni_td_connections = NULL; } // Free the I/O buffers diff --git a/src/base/worker_thread_init.c b/src/base/worker_thread_init.c index 52a09c8a..e404549d 100644 --- a/src/base/worker_thread_init.c +++ b/src/base/worker_thread_init.c @@ -117,7 +117,8 @@ xdd_worker_thread_init(worker_data_t *wdp) { // being present. -nlmills // Request an I/O buffer from XNI - xni_request_target_buffer(tdp->td_e2ep->xni_td_conn, &wdp->wd_e2ep->xni_wd_buf); + xni_request_target_buffer(*xint_e2e_worker_connection(wdp), + &wdp->wd_e2ep->xni_wd_buf); wdp->wd_task.task_datap = xni_target_buffer_data(wdp->wd_e2ep->xni_wd_buf); } } else { diff --git a/src/common/end_to_end.h b/src/common/end_to_end.h index f27889b1..26fe67ad 100644 --- a/src/common/end_to_end.h +++ b/src/common/end_to_end.h @@ -56,7 +56,8 @@ struct xint_e2e { xdd_e2e_ate_t e2e_address_table[E2E_ADDRESS_TABLE_ENTRIES]; // Used by E2E to stripe over multiple IP Addresses /* XNI Target data */ - xni_connection_t xni_td_conn; + xni_connection_t *xni_td_connections; // One connection per host + int xni_td_connections_count; // Number of connection objects /* XNI Worker data */ xni_target_buffer_t xni_wd_buf; diff --git a/src/common/target_data.c b/src/common/target_data.c index 4d5b9cc9..978c099c 100644 --- a/src/common/target_data.c +++ b/src/common/target_data.c @@ -102,6 +102,8 @@ xdd_init_new_target_data(target_data_t *tdp, int32_t n) { tdp->td_e2ep->e2e_dest_addr = 0; tdp->td_e2ep->e2e_wait_1st_msg = 0; tdp->td_e2ep->e2e_address_table_next_entry=0; + tdp->td_e2ep->xni_td_connections = NULL; + tdp->td_e2ep->xni_td_connections_count = 0; } tdp->io_buffers = NULL; diff --git a/src/common/xint_prototypes.h b/src/common/xint_prototypes.h index 66819e89..f817d7da 100644 --- a/src/common/xint_prototypes.h +++ b/src/common/xint_prototypes.h @@ -389,6 +389,7 @@ int32_t xint_e2e_xni_send(worker_data_t *wdp); int32_t xint_e2e_eof_source_side(worker_data_t *wdp); int32_t xint_e2e_xni_recv(worker_data_t *wdp); int xint_is_e2e(const target_data_t *tdp); +xni_connection_t *xint_e2e_worker_connection(worker_data_t *wdp); #endif /* * Local variables: diff --git a/src/xnet/xnet_end_to_end.c b/src/xnet/xnet_end_to_end.c index 284de4d0..1c2b50fc 100644 --- a/src/xnet/xnet_end_to_end.c +++ b/src/xnet/xnet_end_to_end.c @@ -44,6 +44,9 @@ #define de2eprintf(...) #endif +// forward declarations +static xni_connection_t *target_connection(target_data_t*); + int32_t xint_e2e_src_connect(target_data_t *tdp) { int rc = 0; @@ -73,7 +76,7 @@ int32_t xint_e2e_src_connect(target_data_t *tdp) { bufset.bufsize = tdp->io_buffer_size; bufset.reserved = getpagesize(); - rc = xni_connect(tdp->xni_ctx, &xep, &bufset, &tdp->td_e2ep->xni_td_conn); + rc = xni_connect(tdp->xni_ctx, &xep, &bufset, target_connection(tdp)); // translate the error code rc = (XNI_OK == rc) ? 0 : -1; @@ -108,7 +111,7 @@ int32_t xint_e2e_dest_connect(target_data_t *tdp) { bufset.bufsize = tdp->io_buffer_size; bufset.reserved = getpagesize(); - rc = xni_accept_connection(tdp->xni_ctx, &xep, &bufset, &tdp->td_e2ep->xni_td_conn); + rc = xni_accept_connection(tdp->xni_ctx, &xep, &bufset, target_connection(tdp)); // translate the error code rc = (XNI_OK == rc) ? 0 : -1; @@ -126,7 +129,7 @@ int32_t xint_e2e_disconnect(target_data_t *tdp) { //TODO: handle errors - int rc = xni_close_connection(&tdp->td_e2ep->xni_td_conn); + int rc = xni_close_connection(target_connection(tdp)); assert(XNI_OK == rc); return 0; @@ -177,10 +180,12 @@ int32_t xint_e2e_xni_send(worker_data_t *wdp) { nclk_now(&wdp->wd_counters.tc_current_net_start_time); - e2ep->e2e_send_status = xni_send_target_buffer(tdp->td_e2ep->xni_td_conn, + xni_connection_t * const connp = xint_e2e_worker_connection(wdp); + + e2ep->e2e_send_status = xni_send_target_buffer(*connp, &e2ep->xni_wd_buf); // Request a fresh buffer from XNI - xni_request_target_buffer(tdp->td_e2ep->xni_td_conn, &wdp->wd_e2ep->xni_wd_buf); + xni_request_target_buffer(*connp, &wdp->wd_e2ep->xni_wd_buf); // Keep a pointer to the data portion of the buffer wdp->wd_task.task_datap = xni_target_buffer_data(wdp->wd_e2ep->xni_wd_buf); @@ -254,7 +259,7 @@ int32_t xint_e2e_xni_recv(worker_data_t *wdp) { /* Receive a target buffer and assemble it into the wdp */ tdp = wdp->wd_tdp; - status = xni_receive_target_buffer(tdp->td_e2ep->xni_td_conn, + status = xni_receive_target_buffer(*xint_e2e_worker_connection(wdp), &wdp->wd_e2ep->xni_wd_buf); if (XNI_OK == status) { @@ -306,6 +311,30 @@ xint_is_e2e(const target_data_t *tdp) return (TO_ENDTOEND == (TO_ENDTOEND & tdp->td_target_options)); } +xni_connection_t* +xint_e2e_worker_connection(worker_data_t *wdp) +{ + target_data_t * const tdp = wdp->wd_tdp; + + //TODO: support more than one host + assert(1 == tdp->td_e2ep->e2e_address_table_host_count); + xni_connection_t * const conn = tdp->td_e2ep->xni_td_connections; + + return conn; +} + +//TODO: remove this function +// temporary helper function until connection is moved from target +// thread to worker thread +static xni_connection_t* +target_connection(target_data_t *tdp) +{ + assert(1 == tdp->td_e2ep->e2e_address_table_host_count); + xni_connection_t * const conn = tdp->td_e2ep->xni_td_connections; + + return conn; +} + /* * Local variables: * indent-tabs-mode: t diff --git a/src/xnet/xnet_end_to_end_init.c b/src/xnet/xnet_end_to_end_init.c index 03bffb76..2f975577 100644 --- a/src/xnet/xnet_end_to_end_init.c +++ b/src/xnet/xnet_end_to_end_init.c @@ -97,6 +97,15 @@ init_xni(target_data_t *tdp) /* Create the XNI context */ rc = xni_context_create(tdp->xni_pcl, tdp->xni_cb, &tdp->xni_ctx); assert(0 == rc); + + struct xint_e2e * const e2ep = tdp->td_e2ep; + + // Allocate XNI connections, one per e2e host + int conncnt = (int)e2ep->e2e_address_table_host_count; + e2ep->xni_td_connections = calloc(conncnt, + sizeof(*e2ep->xni_td_connections)); + e2ep->xni_td_connections_count = conncnt; + return(0); } From 457b0c83f5c1a0aac362094a8b85280fb0b01f8a Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Wed, 23 Jul 2014 20:12:52 -0400 Subject: [PATCH 34/50] Plug memory leak --- src/base/target_cleanup.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/base/target_cleanup.c b/src/base/target_cleanup.c index 2a59cfd7..1c277014 100644 --- a/src/base/target_cleanup.c +++ b/src/base/target_cleanup.c @@ -48,6 +48,7 @@ xdd_target_thread_cleanup(target_data_t *tdp) { // Free the connections tdp->td_e2ep->xni_td_connections_count = 0; + free(tdp->td_e2ep->xni_td_connections); tdp->td_e2ep->xni_td_connections = NULL; } From 0e6fe983c89cc3cba91c32326a03691f0ad7f704 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Wed, 23 Jul 2014 21:07:25 -0400 Subject: [PATCH 35/50] Init the datapattern buffer after worker init This move is neccessary because soon the buffer won't be valid inside of worker init. This change will also cause the datapattern buffers to be initialized in parallel (whereas before it was serialized inside of worker init). --- src/base/worker_thread.c | 5 +++++ src/base/worker_thread_init.c | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/base/worker_thread.c b/src/base/worker_thread.c index 9d478c80..28fe654f 100644 --- a/src/base/worker_thread.c +++ b/src/base/worker_thread.c @@ -48,6 +48,11 @@ xdd_worker_thread(void *pin) { if ( xgp->abort == 1) // Something went wrong during thread initialization so let's just leave return(0); + // Set the buffer data pattern for non-E2E operations or E2E sources + if (!xint_is_e2e(tdp) || !(tdp->td_target_options & TO_E2E_DESTINATION)) { + xdd_datapattern_buffer_init(wdp); + } + // If this is a dry run then just exit at this point if (xgp->global_options & GO_DRYRUN) return(0); diff --git a/src/base/worker_thread_init.c b/src/base/worker_thread_init.c index e404549d..2ffd7a2a 100644 --- a/src/base/worker_thread_init.c +++ b/src/base/worker_thread_init.c @@ -128,11 +128,6 @@ xdd_worker_thread_init(worker_data_t *wdp) { wdp->wd_task.task_datap = tdp->io_buffers[workernum]; } - // Set the buffer data pattern for non-E2E operations or E2E sources - if (!xint_is_e2e(tdp) || !(tdp->td_target_options & TO_E2E_DESTINATION)) { - xdd_datapattern_buffer_init(wdp); - } - // Init the WorkerThread-TargetPass WAIT Barrier for this WorkerThread sprintf(tmpname,"T%04d:W%04d>worker_thread_targetpass_wait_barrier",tdp->td_target_number,wdp->wd_worker_number); status = xdd_init_barrier(tdp->td_planp, &wdp->wd_thread_targetpass_wait_for_task_barrier, 2, tmpname); From 176d39e06b25d249f676d52013f15d0abb121b61 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Wed, 23 Jul 2014 21:32:26 -0400 Subject: [PATCH 36/50] Allocate source buffers after worker init --- src/base/worker_thread.c | 8 ++++++++ src/base/worker_thread_init.c | 20 ++++---------------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/src/base/worker_thread.c b/src/base/worker_thread.c index 28fe654f..1033d2cd 100644 --- a/src/base/worker_thread.c +++ b/src/base/worker_thread.c @@ -48,6 +48,14 @@ xdd_worker_thread(void *pin) { if ( xgp->abort == 1) // Something went wrong during thread initialization so let's just leave return(0); + //TODO: move this block once the workers establish connections + if (xint_is_e2e(tdp) && !(tdp->td_target_options & TO_E2E_DESTINATION)) { + // Request an I/O buffer from XNI + xni_request_target_buffer(*xint_e2e_worker_connection(wdp), + &wdp->wd_e2ep->xni_wd_buf); + wdp->wd_task.task_datap = xni_target_buffer_data(wdp->wd_e2ep->xni_wd_buf); + } + // Set the buffer data pattern for non-E2E operations or E2E sources if (!xint_is_e2e(tdp) || !(tdp->td_target_options & TO_E2E_DESTINATION)) { xdd_datapattern_buffer_init(wdp); diff --git a/src/base/worker_thread_init.c b/src/base/worker_thread_init.c index 2ffd7a2a..ae668d47 100644 --- a/src/base/worker_thread_init.c +++ b/src/base/worker_thread_init.c @@ -105,22 +105,10 @@ xdd_worker_thread_init(worker_data_t *wdp) { // Set up I/O buffer pointers if (xint_is_e2e(tdp)) { - if (tdp->td_target_options & TO_E2E_DESTINATION) { - // Buffer for destination is set after a receive - wdp->wd_e2ep->xni_wd_buf = NULL; - wdp->wd_task.task_datap = NULL; - } else { - //TODO: I'd really like to move all of these buffer - // request calls to something like ttd_before_io_op, but I - // can't right now because later on in this function - // xdd_datapattern_buffer_init() relies on the buffer - // being present. -nlmills - - // Request an I/O buffer from XNI - xni_request_target_buffer(*xint_e2e_worker_connection(wdp), - &wdp->wd_e2ep->xni_wd_buf); - wdp->wd_task.task_datap = xni_target_buffer_data(wdp->wd_e2ep->xni_wd_buf); - } + // the buffer for source will be requested after connecting + // the buffer for destination is set after a receive + wdp->wd_e2ep->xni_wd_buf = NULL; + wdp->wd_task.task_datap = NULL; } else { // For non-E2E operations the data portion is the entire buffer const int32_t workernum = wdp->wd_worker_number; From b1056691c2cee4f9cdfa91b712bfa8024672274b Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Wed, 23 Jul 2014 22:12:44 -0400 Subject: [PATCH 37/50] Factor out common e2e connect code --- src/xnet/xnet_end_to_end.c | 82 +++++++++++++++----------------------- 1 file changed, 32 insertions(+), 50 deletions(-) diff --git a/src/xnet/xnet_end_to_end.c b/src/xnet/xnet_end_to_end.c index 1c2b50fc..31a35900 100644 --- a/src/xnet/xnet_end_to_end.c +++ b/src/xnet/xnet_end_to_end.c @@ -45,30 +45,33 @@ #endif // forward declarations +static int32_t do_connect(target_data_t*, int); static xni_connection_t *target_connection(target_data_t*); -int32_t xint_e2e_src_connect(target_data_t *tdp) { - +static int32_t +do_connect(target_data_t *tdp, int isdest) +{ int rc = 0; - int e2e_idx = 0; - - /* Loop through the available addresses, and connect */ - while (0 == tdp->td_e2ep->e2e_address_table[e2e_idx].port_count) - e2e_idx++; - - /* Resolve name to an IP */ - rc = xint_lookup_addr(tdp->td_e2ep->e2e_address_table[e2e_idx].hostname, 0, - &tdp->td_e2ep->e2e_dest_addr); + xdd_e2e_ate_t *ate = NULL; // current address table entry + + // Find an available address + for (ate = tdp->td_e2ep->e2e_address_table; 0 == ate->port_count; ate++); + + // Resolve name to an IP address + rc = xint_lookup_addr(ate->hostname, 0, &tdp->td_e2ep->e2e_dest_addr); assert(0 == rc); struct in_addr addr = { .s_addr = tdp->td_e2ep->e2e_dest_addr }; char* ip_string = inet_ntoa(addr); - fprintf(xgp->errout, "Dest host: %s Connect IP: %s Port: %d\n", tdp->td_e2ep->e2e_address_table[e2e_idx].hostname, ip_string, tdp->td_e2ep->e2e_address_table[e2e_idx].base_port); + + if (!isdest) { + fprintf(xgp->errout, "Dest host: %s Connect IP: %s Port: %d\n", + ate->hostname, ip_string, ate->base_port); + } - /* Create an XNI endpoint from the e2e spec */ - xni_endpoint_t xep = {.host = ip_string, - .port = tdp->td_e2ep->e2e_address_table[e2e_idx].base_port}; + // Create an XNI endpoint from the e2e spec + xni_endpoint_t xep = {.host = ip_string, .port = ate->base_port}; - // initialize the set of I/O buffers + // Initialize the set of I/O buffers xni_bufset_t bufset; memset(&bufset, 0, sizeof(bufset)); bufset.bufs = tdp->io_buffers; @@ -76,46 +79,25 @@ int32_t xint_e2e_src_connect(target_data_t *tdp) { bufset.bufsize = tdp->io_buffer_size; bufset.reserved = getpagesize(); - rc = xni_connect(tdp->xni_ctx, &xep, &bufset, target_connection(tdp)); - // translate the error code + if (isdest) { + rc = xni_accept_connection(tdp->xni_ctx, &xep, &bufset, target_connection(tdp)); + } else { + rc = xni_connect(tdp->xni_ctx, &xep, &bufset, target_connection(tdp)); + } + // Translate the error code rc = (XNI_OK == rc) ? 0 : -1; return rc; } -int32_t xint_e2e_dest_connect(target_data_t *tdp) { - - int rc = 0; - int e2e_idx = 0; - - /* Loop through the available addresses, and connect */ - while (0 == tdp->td_e2ep->e2e_address_table[e2e_idx].port_count) - e2e_idx++; - - /* Resolve name to an IP */ - rc = xint_lookup_addr(tdp->td_e2ep->e2e_address_table[e2e_idx].hostname, 0, - &tdp->td_e2ep->e2e_dest_addr); - assert(0 == rc); - struct in_addr addr = { .s_addr = tdp->td_e2ep->e2e_dest_addr }; - char* ip_string = inet_ntoa(addr); - - /* Create an XNI endpoint from the e2e spec */ - xni_endpoint_t xep = {.host = ip_string, - .port = tdp->td_e2ep->e2e_address_table[e2e_idx].base_port}; - - // initialize the set of I/O buffers - xni_bufset_t bufset; - memset(&bufset, 0, sizeof(bufset)); - bufset.bufs = tdp->io_buffers; - bufset.bufcount = tdp->io_buffers_count; - bufset.bufsize = tdp->io_buffer_size; - bufset.reserved = getpagesize(); - - rc = xni_accept_connection(tdp->xni_ctx, &xep, &bufset, target_connection(tdp)); - // translate the error code - rc = (XNI_OK == rc) ? 0 : -1; +int32_t xint_e2e_src_connect(target_data_t *tdp) +{ + return do_connect(tdp, FALSE); +} - return rc; +int32_t xint_e2e_dest_connect(target_data_t *tdp) +{ + return do_connect(tdp, TRUE); } /* From 47b225a5f20dfdf80dd39105bc8e91e2ae4badae Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Thu, 24 Jul 2014 16:57:37 -0400 Subject: [PATCH 38/50] Remove stale comment --- src/base/target_init.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/base/target_init.c b/src/base/target_init.c index 8a425241..e31e02db 100644 --- a/src/base/target_init.c +++ b/src/base/target_init.c @@ -161,8 +161,7 @@ xint_target_init(target_data_t *tdp) { // if this is an end-to-end transfer than perform the connection(s) if (xint_is_e2e(tdp)) { - //TODO: connect with the previously allocated buffers - /* Perform the XNI accept/connect */ + // Perform the XNI accept/connect if (tdp->td_target_options & TO_E2E_DESTINATION) { status = xint_e2e_dest_connect(tdp); } else { From 84f1780aa411bd9cd5d7c21ccb1eb4670cb9e133 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Thu, 24 Jul 2014 17:02:22 -0400 Subject: [PATCH 39/50] Move e2e setup after worker init function --- src/base/worker_thread.c | 14 ++++++++++++++ src/base/worker_thread_init.c | 25 ------------------------- 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/src/base/worker_thread.c b/src/base/worker_thread.c index 1033d2cd..9d1288c5 100644 --- a/src/base/worker_thread.c +++ b/src/base/worker_thread.c @@ -56,6 +56,20 @@ xdd_worker_thread(void *pin) { wdp->wd_task.task_datap = xni_target_buffer_data(wdp->wd_e2ep->xni_wd_buf); } + // Set up for an End-to-End operation (if requested) + if (xint_is_e2e(tdp)) { + status = xint_e2e_worker_init(wdp); + if (-1 == status) { + fprintf(xgp->errout, + "%s: xdd_worker_thread: Target %d WorkerThread %d: E2E %s initialization failed.\n", + xgp->progname, + tdp->td_target_number, + wdp->wd_worker_number, + (tdp->td_target_options & TO_E2E_DESTINATION) ? "DESTINATION":"SOURCE"); + return(0); + } + } + // Set the buffer data pattern for non-E2E operations or E2E sources if (!xint_is_e2e(tdp) || !(tdp->td_target_options & TO_E2E_DESTINATION)) { xdd_datapattern_buffer_init(wdp); diff --git a/src/base/worker_thread_init.c b/src/base/worker_thread_init.c index ae668d47..e0a0792f 100644 --- a/src/base/worker_thread_init.c +++ b/src/base/worker_thread_init.c @@ -153,31 +153,6 @@ xdd_worker_thread_init(worker_data_t *wdp) { errno); } - // Set up for an End-to-End operation (if requested) - if (tdp->td_target_options & TO_ENDTOEND) { - if (tdp->td_target_options & (TO_E2E_DESTINATION|TO_E2E_SOURCE)) { - status = xint_e2e_worker_init(wdp); - } else { // Not sure which side of the E2E this target is supposed to be.... - fprintf(xgp->errout,"%s: xdd_worker_thread_init: Target %d WorkerThread %d: Cannot determine which side of the E2E operation this target is supposed to be.\n", - xgp->progname, - tdp->td_target_number, - wdp->wd_worker_number); - fprintf(xgp->errout,"%s: xdd_worker_thread_init: Check to be sure that the '-e2e issource' or '-e2e isdestination' was specified for this target.\n", - xgp->progname); - fflush(xgp->errout); - return(-1); - } - - if (status == -1) { - fprintf(xgp->errout,"%s: xdd_worker_thread_init: Target %d WorkerThread %d: E2E %s initialization failed.\n", - xgp->progname, - tdp->td_target_number, - wdp->wd_worker_number, - (tdp->td_target_options & TO_E2E_DESTINATION) ? "DESTINATION":"SOURCE"); - return(-1); - } - } // End of end-to-end setup - // All went well... return(0); From 7abb8edc59efcd544ce3143e11a0086c3c9da88a Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Thu, 24 Jul 2014 18:18:45 -0400 Subject: [PATCH 40/50] Establish e2e connections in worker threads Connection establishment was moved to the worker threads from the target thread. This change will allow the workers to have different connections in the future (a requirement for multinic). Currently we still only support a single connection, i.e. one remote host. A mutex protects each connection. The first worker to hold the mutex will establish the connection. The other workers will use the connection that was already established. --- src/base/target_cleanup.c | 19 ++++++++--- src/base/target_init.c | 14 -------- src/base/worker_thread.c | 10 +----- src/common/end_to_end.h | 1 + src/common/target_data.c | 1 + src/common/xint_prototypes.h | 5 +-- src/xnet/xnet_end_to_end.c | 59 +++++++++++++++++++++------------ src/xnet/xnet_end_to_end_init.c | 29 ++++++++++++++-- 8 files changed, 85 insertions(+), 53 deletions(-) diff --git a/src/base/target_cleanup.c b/src/base/target_cleanup.c index 1c277014..8812571d 100644 --- a/src/base/target_cleanup.c +++ b/src/base/target_cleanup.c @@ -42,14 +42,25 @@ xdd_target_thread_cleanup(target_data_t *tdp) { #endif } - // Disconnect if this is an e2e transfer + // if this is an e2e transfer if (xint_is_e2e(tdp)) { + // Disconnect xint_e2e_disconnect(tdp); + struct xint_e2e * const e2ep = tdp->td_e2ep; + // Free the connections - tdp->td_e2ep->xni_td_connections_count = 0; - free(tdp->td_e2ep->xni_td_connections); - tdp->td_e2ep->xni_td_connections = NULL; + e2ep->xni_td_connections_count = 0; + free(e2ep->xni_td_connections); + e2ep->xni_td_connections = NULL; + + // Free the connection mutexes + for (int i = 0; i < e2ep->xni_td_connections_count; i++) { + int error = pthread_mutex_destroy(e2ep->xni_td_connection_mutexes+i); + assert(!error); + } + free(e2ep->xni_td_connection_mutexes); + e2ep->xni_td_connection_mutexes = NULL; } // Free the I/O buffers diff --git a/src/base/target_init.c b/src/base/target_init.c index e31e02db..417d8d68 100644 --- a/src/base/target_init.c +++ b/src/base/target_init.c @@ -159,20 +159,6 @@ xint_target_init(target_data_t *tdp) { return -1; } - // if this is an end-to-end transfer than perform the connection(s) - if (xint_is_e2e(tdp)) { - // Perform the XNI accept/connect - if (tdp->td_target_options & TO_E2E_DESTINATION) { - status = xint_e2e_dest_connect(tdp); - } else { - status = xint_e2e_src_connect(tdp); - } - if (0 != status) { - fprintf(xgp->errout, "Failure during XNI connection.\n"); - return -1; - } - } - // Start the WorkerThreads status = xint_target_init_start_worker_threads(tdp); if (status) diff --git a/src/base/worker_thread.c b/src/base/worker_thread.c index 9d1288c5..31bc01cb 100644 --- a/src/base/worker_thread.c +++ b/src/base/worker_thread.c @@ -48,16 +48,8 @@ xdd_worker_thread(void *pin) { if ( xgp->abort == 1) // Something went wrong during thread initialization so let's just leave return(0); - //TODO: move this block once the workers establish connections - if (xint_is_e2e(tdp) && !(tdp->td_target_options & TO_E2E_DESTINATION)) { - // Request an I/O buffer from XNI - xni_request_target_buffer(*xint_e2e_worker_connection(wdp), - &wdp->wd_e2ep->xni_wd_buf); - wdp->wd_task.task_datap = xni_target_buffer_data(wdp->wd_e2ep->xni_wd_buf); - } - - // Set up for an End-to-End operation (if requested) if (xint_is_e2e(tdp)) { + // Set up for an e2e operation and establish a connection status = xint_e2e_worker_init(wdp); if (-1 == status) { fprintf(xgp->errout, diff --git a/src/common/end_to_end.h b/src/common/end_to_end.h index 26fe67ad..80239c38 100644 --- a/src/common/end_to_end.h +++ b/src/common/end_to_end.h @@ -58,6 +58,7 @@ struct xint_e2e { /* XNI Target data */ xni_connection_t *xni_td_connections; // One connection per host int xni_td_connections_count; // Number of connection objects + pthread_mutex_t *xni_td_connection_mutexes; // To synchronize connection establishment; one per connection /* XNI Worker data */ xni_target_buffer_t xni_wd_buf; diff --git a/src/common/target_data.c b/src/common/target_data.c index 978c099c..3711c7c3 100644 --- a/src/common/target_data.c +++ b/src/common/target_data.c @@ -104,6 +104,7 @@ xdd_init_new_target_data(target_data_t *tdp, int32_t n) { tdp->td_e2ep->e2e_address_table_next_entry=0; tdp->td_e2ep->xni_td_connections = NULL; tdp->td_e2ep->xni_td_connections_count = 0; + tdp->td_e2ep->xni_td_connection_mutexes = NULL; } tdp->io_buffers = NULL; diff --git a/src/common/xint_prototypes.h b/src/common/xint_prototypes.h index f817d7da..79fec90f 100644 --- a/src/common/xint_prototypes.h +++ b/src/common/xint_prototypes.h @@ -382,14 +382,15 @@ int32_t xint_e2e_worker_init(worker_data_t *wdp); xint_e2e_t *xint_get_e2ep(void); // xnet_end_to_end.c -int32_t xint_e2e_dest_connect(target_data_t *tdp); -int32_t xint_e2e_src_connect(target_data_t *tdp); +int32_t xint_e2e_dest_connect(worker_data_t *wdp); +int32_t xint_e2e_src_connect(worker_data_t *wdp); int32_t xint_e2e_disconnect(target_data_t *tdp); int32_t xint_e2e_xni_send(worker_data_t *wdp); int32_t xint_e2e_eof_source_side(worker_data_t *wdp); int32_t xint_e2e_xni_recv(worker_data_t *wdp); int xint_is_e2e(const target_data_t *tdp); xni_connection_t *xint_e2e_worker_connection(worker_data_t *wdp); +pthread_mutex_t *xint_e2e_worker_connection_mutex(worker_data_t *wdp); #endif /* * Local variables: diff --git a/src/xnet/xnet_end_to_end.c b/src/xnet/xnet_end_to_end.c index 31a35900..b90df127 100644 --- a/src/xnet/xnet_end_to_end.c +++ b/src/xnet/xnet_end_to_end.c @@ -45,14 +45,25 @@ #endif // forward declarations -static int32_t do_connect(target_data_t*, int); -static xni_connection_t *target_connection(target_data_t*); +static int32_t do_connect(worker_data_t*, int); static int32_t -do_connect(target_data_t *tdp, int isdest) +do_connect(worker_data_t *wdp, int isdest) { + pthread_mutex_t * const mutex = xint_e2e_worker_connection_mutex(wdp); + (void)pthread_mutex_lock(mutex); + + xni_connection_t * const conn = xint_e2e_worker_connection(wdp); + //TODO: find a safer test (maybe add something to XNI?) + if (*conn) { + // Bail; some other worker has already connected + (void)pthread_mutex_unlock(mutex); + return 0; + } + int rc = 0; xdd_e2e_ate_t *ate = NULL; // current address table entry + target_data_t * const tdp = wdp->wd_tdp; // Find an available address for (ate = tdp->td_e2ep->e2e_address_table; 0 == ate->port_count; ate++); @@ -80,24 +91,25 @@ do_connect(target_data_t *tdp, int isdest) bufset.reserved = getpagesize(); if (isdest) { - rc = xni_accept_connection(tdp->xni_ctx, &xep, &bufset, target_connection(tdp)); + rc = xni_accept_connection(tdp->xni_ctx, &xep, &bufset, conn); } else { - rc = xni_connect(tdp->xni_ctx, &xep, &bufset, target_connection(tdp)); + rc = xni_connect(tdp->xni_ctx, &xep, &bufset, conn); } // Translate the error code rc = (XNI_OK == rc) ? 0 : -1; + (void)pthread_mutex_unlock(mutex); return rc; } -int32_t xint_e2e_src_connect(target_data_t *tdp) +int32_t xint_e2e_src_connect(worker_data_t *wdp) { - return do_connect(tdp, FALSE); + return do_connect(wdp, FALSE); } -int32_t xint_e2e_dest_connect(target_data_t *tdp) +int32_t xint_e2e_dest_connect(worker_data_t *wdp) { - return do_connect(tdp, TRUE); + return do_connect(wdp, TRUE); } /* @@ -110,9 +122,14 @@ int32_t xint_e2e_dest_connect(target_data_t *tdp) int32_t xint_e2e_disconnect(target_data_t *tdp) { - //TODO: handle errors - int rc = xni_close_connection(target_connection(tdp)); - assert(XNI_OK == rc); + xint_e2e_t * const e2ep = tdp->td_e2ep; + + // Close all connections + for (int i = 0; i < e2ep->xni_td_connections_count; i++) { + int rc = xni_close_connection(e2ep->xni_td_connections+i); + //TODO: handle errors + assert(XNI_OK == rc); + } return 0; } @@ -305,16 +322,16 @@ xint_e2e_worker_connection(worker_data_t *wdp) return conn; } -//TODO: remove this function -// temporary helper function until connection is moved from target -// thread to worker thread -static xni_connection_t* -target_connection(target_data_t *tdp) -{ - assert(1 == tdp->td_e2ep->e2e_address_table_host_count); - xni_connection_t * const conn = tdp->td_e2ep->xni_td_connections; +pthread_mutex_t* +xint_e2e_worker_connection_mutex(worker_data_t *wdp) +{ + xint_e2e_t * const e2ep = wdp->wd_tdp->td_e2ep; - return conn; + //TODO: support more than one host + assert(1 == e2ep->e2e_address_table_host_count); + pthread_mutex_t * const mutex = e2ep->xni_td_connection_mutexes; + + return mutex; } /* diff --git a/src/xnet/xnet_end_to_end_init.c b/src/xnet/xnet_end_to_end_init.c index 2f975577..c1101f98 100644 --- a/src/xnet/xnet_end_to_end_init.c +++ b/src/xnet/xnet_end_to_end_init.c @@ -99,9 +99,17 @@ init_xni(target_data_t *tdp) assert(0 == rc); struct xint_e2e * const e2ep = tdp->td_e2ep; + const int conncnt = (int)e2ep->e2e_address_table_host_count; + + // Initialize mutexes that protect connection establishment + e2ep->xni_td_connection_mutexes = calloc(conncnt, + sizeof(*e2ep->xni_td_connection_mutexes)); + for (int i = 0; i < conncnt; i++) { + rc = pthread_mutex_init(e2ep->xni_td_connection_mutexes+i, NULL); + assert(0 == rc); + } // Allocate XNI connections, one per e2e host - int conncnt = (int)e2ep->e2e_address_table_host_count; e2ep->xni_td_connections = calloc(conncnt, sizeof(*e2ep->xni_td_connections)); e2ep->xni_td_connections_count = conncnt; @@ -184,8 +192,18 @@ init_src_worker(worker_data_t *wdp) // Init the relevant variables e2ep->e2e_msg_sequence_number = 0; - return(0); + int status = xint_e2e_src_connect(wdp); + if (0 != status) { + fprintf(xgp->errout, "Failure during XNI connection.\n"); + return -1; + } + + // Request an I/O buffer from XNI + xni_request_target_buffer(*xint_e2e_worker_connection(wdp), + &wdp->wd_e2ep->xni_wd_buf); + wdp->wd_task.task_datap = xni_target_buffer_data(wdp->wd_e2ep->xni_wd_buf); + return(0); } static int @@ -212,8 +230,13 @@ init_dest_worker(worker_data_t *wdp) // Clear the end-of-file flag wdp->wd_e2ep->received_eof = FALSE; - return(0); + int status = xint_e2e_dest_connect(wdp); + if (0 != status) { + fprintf(xgp->errout, "Failure during XNI connection.\n"); + return -1; + } + return(0); } /*----------------------------------------------------------------------------*/ From 16e716fcd49fa3678db8821db1442e0851b3978f Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Fri, 25 Jul 2014 19:00:37 -0400 Subject: [PATCH 41/50] Remove duplicate structure field --- src/client/parse_func.c | 62 +++++++++++++++++++--------------------- src/common/debug.c | 1 - src/common/end_to_end.h | 1 - src/common/target_data.c | 1 - 4 files changed, 30 insertions(+), 35 deletions(-) diff --git a/src/client/parse_func.c b/src/client/parse_func.c index 0f2d181b..f35d9fcc 100644 --- a/src/client/parse_func.c +++ b/src/client/parse_func.c @@ -1015,23 +1015,23 @@ xddfunc_endtoend(xdd_plan_t *planp, int32_t argc, char *argv[], uint32_t flags) if (tdp == NULL) return(-1); tdp->td_target_options |= TO_ENDTOEND; - strcpy(tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].hostname, hostname); - tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].base_port = DEFAULT_E2E_PORT; - tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].port_count = 0; + strcpy(tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].hostname, hostname); + tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].base_port = DEFAULT_E2E_PORT; + tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].port_count = 0; // Set a default NUMA node value if possible #if defined(HAVE_CPU_SET_T) - CPU_ZERO(&tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].cpu_set); + CPU_ZERO(&tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].cpu_set); sched_getaffinity(getpid(), - sizeof(tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].cpu_set), - &tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].cpu_set); + sizeof(tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].cpu_set), + &tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].cpu_set); #endif if (base_port) { // Set the requested Port Number and possible Port Count - tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].base_port = atoi(base_port); + tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].base_port = atoi(base_port); if (port_count) { - tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].port_count = atoi(port_count); + tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].port_count = atoi(port_count); } else { - tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].port_count = 0; + tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].port_count = 0; } #if defined(HAVE_CPU_SET_T) && defined(HAVE_NUMA_NODE_TO_CPUS) && defined(HAVE_NUMA_ALLOCATE_CPUMASK) @@ -1039,18 +1039,18 @@ xddfunc_endtoend(xdd_plan_t *planp, int32_t argc, char *argv[], uint32_t flags) int i; struct bitmask* numa_mask = numa_allocate_cpumask(); int numa_node_no = atoi(numa_node); - CPU_ZERO(&tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].cpu_set); + CPU_ZERO(&tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].cpu_set); numa_node_to_cpus(numa_node_no, numa_mask); for (i = 0; i <= CPU_SETSIZE; i++) { if (numa_bitmask_isbitset(numa_mask, i)) - CPU_SET(i, &tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].cpu_set); + CPU_SET(i, &tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].cpu_set); } numa_free_cpumask(numa_mask); } else { - CPU_ZERO(&tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].cpu_set); + CPU_ZERO(&tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].cpu_set); sched_getaffinity(getpid(), - sizeof(tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].cpu_set), - &tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].cpu_set); + sizeof(tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].cpu_set), + &tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].cpu_set); } #else if (numa_node) { @@ -1058,8 +1058,7 @@ xddfunc_endtoend(xdd_plan_t *planp, int32_t argc, char *argv[], uint32_t flags) } #endif } - tdp->td_e2ep->e2e_address_table_port_count += tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].port_count; - tdp->td_e2ep->e2e_address_table_next_entry++; + tdp->td_e2ep->e2e_address_table_port_count += tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].port_count; tdp->td_e2ep->e2e_address_table_host_count++; } else { /* set option for all targets */ if (flags & XDD_PARSE_PHASE2) { @@ -1067,41 +1066,41 @@ xddfunc_endtoend(xdd_plan_t *planp, int32_t argc, char *argv[], uint32_t flags) i = 0; while (tdp) { tdp->td_target_options |= TO_ENDTOEND; - strcpy(tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].hostname, hostname); - tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].base_port = DEFAULT_E2E_PORT; - tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].port_count = 0; + strcpy(tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].hostname, hostname); + tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].base_port = DEFAULT_E2E_PORT; + tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].port_count = 0; // Set a default NUMA node value if possible #if defined(HAVE_CPU_SET_T) - CPU_ZERO(&tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].cpu_set); + CPU_ZERO(&tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].cpu_set); sched_getaffinity(getpid(), - sizeof(tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].cpu_set), - &tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].cpu_set); + sizeof(tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].cpu_set), + &tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].cpu_set); #endif if (base_port) { // Set the requested Port Number and possible Port Count - tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].base_port = atoi(base_port); + tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].base_port = atoi(base_port); if (port_count) { - tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].port_count = atoi(port_count); + tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].port_count = atoi(port_count); } else { - tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].port_count = 0; + tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].port_count = 0; } #if defined(HAVE_CPU_SET_T) && defined(HAVE_NUMA_NODE_TO_CPUS) && defined(HAVE_NUMA_ALLOCATE_CPUMASK) if (numa_node && -1 != numa_available()) { int i; struct bitmask* numa_mask = numa_allocate_cpumask(); int numa_node_no = atoi(numa_node); - CPU_ZERO(&tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].cpu_set); + CPU_ZERO(&tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].cpu_set); numa_node_to_cpus(numa_node_no, numa_mask); for (i = 0; i <= CPU_SETSIZE; i++) { if (numa_bitmask_isbitset(numa_mask, i)) - CPU_SET(i, &tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].cpu_set); + CPU_SET(i, &tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].cpu_set); } numa_free_cpumask(numa_mask); } else { - CPU_ZERO(&tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].cpu_set); + CPU_ZERO(&tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].cpu_set); sched_getaffinity(getpid(), - sizeof(tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].cpu_set), - &tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].cpu_set); + sizeof(tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].cpu_set), + &tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].cpu_set); } #else if (numa_node) { @@ -1109,9 +1108,8 @@ xddfunc_endtoend(xdd_plan_t *planp, int32_t argc, char *argv[], uint32_t flags) } #endif } // End of IF stmnt that sets the Port/NPorts - tdp->td_e2ep->e2e_address_table_port_count += tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_next_entry].port_count; + tdp->td_e2ep->e2e_address_table_port_count += tdp->td_e2ep->e2e_address_table[tdp->td_e2ep->e2e_address_table_host_count].port_count; tdp->td_e2ep->e2e_address_table_host_count++; - tdp->td_e2ep->e2e_address_table_next_entry++; i++; tdp = planp->target_datap[i]; } diff --git a/src/common/debug.c b/src/common/debug.c index a126cb76..5cc13643 100644 --- a/src/common/debug.c +++ b/src/common/debug.c @@ -395,7 +395,6 @@ xdd_show_e2e(xint_e2e_t *e2ep) { fprintf(stderr,"\txdd_show_e2e: nclk_t e2e_sr_time=%lld\n",(unsigned long long int)e2ep->e2e_sr_time); // Time spent sending or receiving data for End-to-End operation fprintf(stderr,"\txdd_show_e2e: int32_t e2e_address_table_host_count=%d\n",e2ep->e2e_address_table_host_count); // Cumulative number of hosts represented in the e2e address table fprintf(stderr,"\txdd_show_e2e: int32_t e2e_address_table_port_count=%d\n",e2ep->e2e_address_table_port_count); // Cumulative number of ports represented in the e2e address table - fprintf(stderr,"\txdd_show_e2e: int32_t e2e_address_table_next_entry=%d\n",e2ep->e2e_address_table_next_entry); // Next available entry in the e2e_address_table fprintf(stderr,"\txdd_show_e2e: xdd_e2e_ate_t e2e_address_table[E2E_ADDRESS_TABLE_ENTRIES]\n"); // Used by E2E to stripe over multiple IP Addresses fprintf(stderr,"xdd_show_e2e:********* End of E2E Data at 0x%p **********\n",e2ep); } // End of xdd_show_e2e() diff --git a/src/common/end_to_end.h b/src/common/end_to_end.h index 80239c38..c34f7f30 100644 --- a/src/common/end_to_end.h +++ b/src/common/end_to_end.h @@ -52,7 +52,6 @@ struct xint_e2e { nclk_t e2e_sr_time; // Time spent sending or receiving data for End-to-End operation int32_t e2e_address_table_host_count; // Cumulative number of hosts represented in the e2e address table int32_t e2e_address_table_port_count; // Cumulative number of ports represented in the e2e address table - int32_t e2e_address_table_next_entry; // Next available entry in the e2e_address_table xdd_e2e_ate_t e2e_address_table[E2E_ADDRESS_TABLE_ENTRIES]; // Used by E2E to stripe over multiple IP Addresses /* XNI Target data */ diff --git a/src/common/target_data.c b/src/common/target_data.c index 3711c7c3..7b94688f 100644 --- a/src/common/target_data.c +++ b/src/common/target_data.c @@ -101,7 +101,6 @@ xdd_init_new_target_data(target_data_t *tdp, int32_t n) { tdp->td_e2ep->e2e_address_table_port_count = 0; tdp->td_e2ep->e2e_dest_addr = 0; tdp->td_e2ep->e2e_wait_1st_msg = 0; - tdp->td_e2ep->e2e_address_table_next_entry=0; tdp->td_e2ep->xni_td_connections = NULL; tdp->td_e2ep->xni_td_connections_count = 0; tdp->td_e2ep->xni_td_connection_mutexes = NULL; From 70c8d2b8f1c495f207225cdcaa75a44765e7b187 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Fri, 25 Jul 2014 19:27:18 -0400 Subject: [PATCH 42/50] Remove unused fields from e2e structure --- src/base/target_init.c | 5 ++--- src/client/info_display.c | 1 - src/client/parse_func.c | 2 -- src/common/debug.c | 3 --- src/common/end_to_end.h | 3 --- src/common/target_data.c | 2 -- 6 files changed, 2 insertions(+), 14 deletions(-) diff --git a/src/base/target_init.c b/src/base/target_init.c index 417d8d68..1cebe441 100644 --- a/src/base/target_init.c +++ b/src/base/target_init.c @@ -283,7 +283,6 @@ xint_target_init_start_worker_threads(target_data_t *tdp) { //assert(e2e_addr_index < p->e2ep->e2e_address_table->number_of_entries); wdp->wd_e2ep->e2e_dest_hostname = tdp->td_e2ep->e2e_address_table[e2e_addr_index].hostname; - wdp->wd_e2ep->e2e_dest_port = tdp->td_e2ep->e2e_address_table[e2e_addr_index].base_port + e2e_addr_port; // Set the WorkerThread Numa node if possible #if defined(HAVE_CPU_SET_T) && defined(HAVE_PTHREAD_ATTR_SETAFFINITY_NP) @@ -298,9 +297,9 @@ xint_target_init_start_worker_threads(target_data_t *tdp) { e2e_addr_port = 0; } if (xgp->global_options & GO_REALLYVERBOSE) - fprintf(stderr,"Target Init: Target %d: assigning hostname %s port %d to worker_thread %d\n", + fprintf(stderr,"Target Init: Target %d: assigning hostname %s to worker_thread %d\n", tdp->td_target_number, wdp->wd_e2ep->e2e_dest_hostname, - wdp->wd_e2ep->e2e_dest_port, wdp->wd_worker_number); + wdp->wd_worker_number); } diff --git a/src/client/info_display.c b/src/client/info_display.c index 3e8d168c..38ad1428 100644 --- a/src/client/info_display.c +++ b/src/client/info_display.c @@ -421,7 +421,6 @@ xdd_target_info(FILE *out, target_data_t *tdp) { return; } // ok - we have a good restart structure - tdp->td_restartp->source_host = tdp->td_e2ep->e2e_src_hostname; // Name of the Source machine tdp->td_restartp->destination_host = tdp->td_e2ep->e2e_dest_hostname; // Name of the Destination machine if (tdp->td_restartp->flags & RESTART_FLAG_ISSOURCE) { // This is the SOURCE sside of the biz tdp->td_restartp->source_filename = tdp->td_target_full_pathname; // The source_filename is the name of the file being copied on the source side diff --git a/src/client/parse_func.c b/src/client/parse_func.c index f35d9fcc..f3745ccd 100644 --- a/src/client/parse_func.c +++ b/src/client/parse_func.c @@ -3171,7 +3171,6 @@ xddfunc_restart(xdd_plan_t *planp, int32_t argc, char *argv[], uint32_t flags) rp->initial_restart_offset = atoll(argv[args_index+1]); rp->byte_offset = rp->initial_restart_offset; rp->flags |= RESTART_FLAG_RESUME_COPY; - tdp->td_e2ep->e2e_total_bytes_written=rp->byte_offset; /* Set the last committed location to avoid restart output of 0 if the target does not complete any I/O during first interval @@ -3193,7 +3192,6 @@ xddfunc_restart(xdd_plan_t *planp, int32_t argc, char *argv[], uint32_t flags) rp->initial_restart_offset = atoll(argv[args_index+1]); rp->byte_offset = rp->initial_restart_offset; rp->flags |= RESTART_FLAG_RESUME_COPY; - tdp->td_e2ep->e2e_total_bytes_written=rp->byte_offset; /* Set the last committed location to avoid restart output of 0 if the target does not complete any I/O during diff --git a/src/common/debug.c b/src/common/debug.c index 5cc13643..ae45b1aa 100644 --- a/src/common/debug.c +++ b/src/common/debug.c @@ -383,14 +383,11 @@ void xdd_show_e2e(xint_e2e_t *e2ep) { fprintf(stderr,"\nxdd_show_e2e:********* Start of E2E Data at 0x%p **********\n",e2ep); fprintf(stderr,"\txdd_show_e2e: char *e2e_dest_hostname='%s'\n",e2ep->e2e_dest_hostname); // Name of the Destination machine - fprintf(stderr,"\txdd_show_e2e: char *e2e_src_hostname='%s'\n",e2ep->e2e_dest_hostname); // Name of the Source machine fprintf(stderr,"\txdd_show_e2e: char *e2e_src_file_path='%s'\n",e2ep->e2e_dest_hostname); // Full path of source file for destination restart file fprintf(stderr,"\txdd_show_e2e: time_t e2e_src_file_mtime\n"); // stat -c %Y *e2e_src_file_path, i.e., last modification time fprintf(stderr,"\txdd_show_e2e: in_addr_t e2e_dest_addr=%d\n",e2ep->e2e_dest_addr); // Destination Address number of the E2E socket - fprintf(stderr,"\txdd_show_e2e: in_port_t e2e_dest_port=%d\n",e2ep->e2e_dest_port); // Port number to use for the E2E socket fprintf(stderr,"\txdd_show_e2e: int32_t e2e_send_status=%d\n",e2ep->e2e_send_status); // Current Send Status fprintf(stderr,"\txdd_show_e2e: int64_t e2e_msg_sequence_number=%lld\n",(long long int)e2ep->e2e_msg_sequence_number);// The Message Sequence Number of the most recent message sent or to be received - fprintf(stderr,"\txdd_show_e2e: int64_t e2e_total_bytes_written=%lld\n",(long long int)e2ep->e2e_total_bytes_written); // The total amount of data written across all restarts for this file fprintf(stderr,"\txdd_show_e2e: nclk_t e2e_wait_1st_msg=%lld\n",(unsigned long long int)e2ep->e2e_wait_1st_msg); // Time in nanosecs destination waited for 1st source data to arrive fprintf(stderr,"\txdd_show_e2e: nclk_t e2e_sr_time=%lld\n",(unsigned long long int)e2ep->e2e_sr_time); // Time spent sending or receiving data for End-to-End operation fprintf(stderr,"\txdd_show_e2e: int32_t e2e_address_table_host_count=%d\n",e2ep->e2e_address_table_host_count); // Cumulative number of hosts represented in the e2e address table diff --git a/src/common/end_to_end.h b/src/common/end_to_end.h index c34f7f30..d3c5ce7a 100644 --- a/src/common/end_to_end.h +++ b/src/common/end_to_end.h @@ -42,12 +42,9 @@ typedef struct xdd_e2e_address_table xdd_e2e_at_t; */ struct xint_e2e { char *e2e_dest_hostname; // Name of the Destination machine - char *e2e_src_hostname; // Name of the Source machine in_addr_t e2e_dest_addr; // Destination Address number of the E2E socket - in_port_t e2e_dest_port; // Port number to use for the E2E socket int32_t e2e_send_status; // Current Send Status int64_t e2e_msg_sequence_number;// The Message Sequence Number of the most recent message sent or to be received - int64_t e2e_total_bytes_written; // The total amount of data written across all restarts for this file nclk_t e2e_wait_1st_msg; // Time in nanosecs destination waited for 1st source data to arrive nclk_t e2e_sr_time; // Time spent sending or receiving data for End-to-End operation int32_t e2e_address_table_host_count; // Cumulative number of hosts represented in the e2e address table diff --git a/src/common/target_data.c b/src/common/target_data.c index 7b94688f..c1ed90d5 100644 --- a/src/common/target_data.c +++ b/src/common/target_data.c @@ -94,9 +94,7 @@ xdd_init_new_target_data(target_data_t *tdp, int32_t n) { } /* Init the end-to-end fields */ if (tdp->td_e2ep) { - tdp->td_e2ep->e2e_src_hostname = NULL; /* E2E source hostname */ tdp->td_e2ep->e2e_dest_hostname = NULL; /* E2E destination hostname */ - tdp->td_e2ep->e2e_dest_port = DEFAULT_E2E_PORT; tdp->td_e2ep->e2e_address_table_host_count = 0; tdp->td_e2ep->e2e_address_table_port_count = 0; tdp->td_e2ep->e2e_dest_addr = 0; From 39a0700e6a720144e1cb90bfe46ce6ae37485c0b Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Fri, 25 Jul 2014 21:24:11 -0400 Subject: [PATCH 43/50] Workers access e2e data directly from the address table --- src/base/target_init.c | 21 +++++++++++++-------- src/client/info_display.c | 8 +++++++- src/common/debug.c | 2 -- src/common/end_to_end.h | 2 +- src/common/target_data.c | 2 +- src/common/xint_prototypes.h | 1 + src/xnet/xnet_end_to_end.c | 21 +++++++++++++++++++++ src/xnet/xnet_end_to_end_init.c | 6 +++--- 8 files changed, 47 insertions(+), 16 deletions(-) diff --git a/src/base/target_init.c b/src/base/target_init.c index 1cebe441..9610a5f2 100644 --- a/src/base/target_init.c +++ b/src/base/target_init.c @@ -281,8 +281,17 @@ xint_target_init_start_worker_threads(target_data_t *tdp) { e2e_addr_index++; } //assert(e2e_addr_index < p->e2ep->e2e_address_table->number_of_entries); - - wdp->wd_e2ep->e2e_dest_hostname = tdp->td_e2ep->e2e_address_table[e2e_addr_index].hostname; + + if (xgp->global_options & GO_REALLYVERBOSE) { + fprintf(xgp->errout, + "Target Init: Target %d: assigning hostname %s to worker_thread %d\n", + tdp->td_target_number, + tdp->td_e2ep->e2e_address_table[e2e_addr_index].hostname, + wdp->wd_worker_number); + } + + // Assign this address to the current worker + wdp->wd_e2ep->address_table_index = e2e_addr_index; // Set the WorkerThread Numa node if possible #if defined(HAVE_CPU_SET_T) && defined(HAVE_PTHREAD_ATTR_SETAFFINITY_NP) @@ -296,13 +305,9 @@ xint_target_init_start_worker_threads(target_data_t *tdp) { e2e_addr_index++; e2e_addr_port = 0; } - if (xgp->global_options & GO_REALLYVERBOSE) - fprintf(stderr,"Target Init: Target %d: assigning hostname %s to worker_thread %d\n", - tdp->td_target_number, wdp->wd_e2ep->e2e_dest_hostname, - wdp->wd_worker_number); - } - + } // end of e2e initialization + status = pthread_create(&wdp->wd_thread, &worker_thread_attr, xdd_worker_thread, wdp); if (status) { fprintf(xgp->errout,"%s: xdd_target_init_start_worker_threads: ERROR: Cannot create worker_thread %d for target number %d name '%s' - Error number %d\n", diff --git a/src/client/info_display.c b/src/client/info_display.c index 38ad1428..f5f72911 100644 --- a/src/client/info_display.c +++ b/src/client/info_display.c @@ -421,7 +421,13 @@ xdd_target_info(FILE *out, target_data_t *tdp) { return; } // ok - we have a good restart structure - tdp->td_restartp->destination_host = tdp->td_e2ep->e2e_dest_hostname; // Name of the Destination machine + + // Commented out the next line because I removed the + // e2e_dest_hostname field. However, this line couldn't + // have worked anyway because td_e2ep->e2e_dest_hostname + // was never assigned to. Restart has probably been broken + // for some time. -nlmills + //tdp->td_restartp->destination_host = tdp->td_e2ep->e2e_dest_hostname; // Name of the Destination machine if (tdp->td_restartp->flags & RESTART_FLAG_ISSOURCE) { // This is the SOURCE sside of the biz tdp->td_restartp->source_filename = tdp->td_target_full_pathname; // The source_filename is the name of the file being copied on the source side tdp->td_restartp->destination_filename = NULL; // The destination_filename is the name of the file being copied on the destination side diff --git a/src/common/debug.c b/src/common/debug.c index ae45b1aa..ee91ed3d 100644 --- a/src/common/debug.c +++ b/src/common/debug.c @@ -382,8 +382,6 @@ xdd_show_target_counters(xint_target_counters_t *tcp) { void xdd_show_e2e(xint_e2e_t *e2ep) { fprintf(stderr,"\nxdd_show_e2e:********* Start of E2E Data at 0x%p **********\n",e2ep); - fprintf(stderr,"\txdd_show_e2e: char *e2e_dest_hostname='%s'\n",e2ep->e2e_dest_hostname); // Name of the Destination machine - fprintf(stderr,"\txdd_show_e2e: char *e2e_src_file_path='%s'\n",e2ep->e2e_dest_hostname); // Full path of source file for destination restart file fprintf(stderr,"\txdd_show_e2e: time_t e2e_src_file_mtime\n"); // stat -c %Y *e2e_src_file_path, i.e., last modification time fprintf(stderr,"\txdd_show_e2e: in_addr_t e2e_dest_addr=%d\n",e2ep->e2e_dest_addr); // Destination Address number of the E2E socket fprintf(stderr,"\txdd_show_e2e: int32_t e2e_send_status=%d\n",e2ep->e2e_send_status); // Current Send Status diff --git a/src/common/end_to_end.h b/src/common/end_to_end.h index d3c5ce7a..7d255b0f 100644 --- a/src/common/end_to_end.h +++ b/src/common/end_to_end.h @@ -41,7 +41,6 @@ typedef struct xdd_e2e_address_table xdd_e2e_at_t; * target thread and worker thread. */ struct xint_e2e { - char *e2e_dest_hostname; // Name of the Destination machine in_addr_t e2e_dest_addr; // Destination Address number of the E2E socket int32_t e2e_send_status; // Current Send Status int64_t e2e_msg_sequence_number;// The Message Sequence Number of the most recent message sent or to be received @@ -59,6 +58,7 @@ struct xint_e2e { /* XNI Worker data */ xni_target_buffer_t xni_wd_buf; int received_eof; // TRUE when the source has signaled EOF to this destination worker + int address_table_index; // Index into e2e_address_table for the address assigned to this worker }; // End of struct xint_e2e definition typedef struct xint_e2e xint_e2e_t; diff --git a/src/common/target_data.c b/src/common/target_data.c index c1ed90d5..ade793a3 100644 --- a/src/common/target_data.c +++ b/src/common/target_data.c @@ -94,7 +94,6 @@ xdd_init_new_target_data(target_data_t *tdp, int32_t n) { } /* Init the end-to-end fields */ if (tdp->td_e2ep) { - tdp->td_e2ep->e2e_dest_hostname = NULL; /* E2E destination hostname */ tdp->td_e2ep->e2e_address_table_host_count = 0; tdp->td_e2ep->e2e_address_table_port_count = 0; tdp->td_e2ep->e2e_dest_addr = 0; @@ -102,6 +101,7 @@ xdd_init_new_target_data(target_data_t *tdp, int32_t n) { tdp->td_e2ep->xni_td_connections = NULL; tdp->td_e2ep->xni_td_connections_count = 0; tdp->td_e2ep->xni_td_connection_mutexes = NULL; + tdp->td_e2ep->address_table_index = -1; } tdp->io_buffers = NULL; diff --git a/src/common/xint_prototypes.h b/src/common/xint_prototypes.h index 79fec90f..40885e7c 100644 --- a/src/common/xint_prototypes.h +++ b/src/common/xint_prototypes.h @@ -391,6 +391,7 @@ int32_t xint_e2e_xni_recv(worker_data_t *wdp); int xint_is_e2e(const target_data_t *tdp); xni_connection_t *xint_e2e_worker_connection(worker_data_t *wdp); pthread_mutex_t *xint_e2e_worker_connection_mutex(worker_data_t *wdp); +const char *xint_e2e_worker_dest_hostname(worker_data_t *wdp); #endif /* * Local variables: diff --git a/src/xnet/xnet_end_to_end.c b/src/xnet/xnet_end_to_end.c index b90df127..5c027e64 100644 --- a/src/xnet/xnet_end_to_end.c +++ b/src/xnet/xnet_end_to_end.c @@ -46,6 +46,7 @@ // forward declarations static int32_t do_connect(worker_data_t*, int); +static xdd_e2e_ate_t *worker_address_table_entry(worker_data_t*); static int32_t do_connect(worker_data_t *wdp, int isdest) @@ -334,6 +335,26 @@ xint_e2e_worker_connection_mutex(worker_data_t *wdp) return mutex; } +static xdd_e2e_ate_t* +worker_address_table_entry(worker_data_t *wdp) +{ + const int idx = wdp->wd_e2ep->address_table_index; + xdd_e2e_ate_t * const atep = idx >= 0 + ? wdp->wd_tdp->td_e2ep->e2e_address_table+idx + : NULL; + + return atep; +} + +const char* +xint_e2e_worker_dest_hostname(worker_data_t *wdp) +{ + const xdd_e2e_ate_t * const atep = worker_address_table_entry(wdp); + const char *hostname = atep->hostname; + + return hostname; +} + /* * Local variables: * indent-tabs-mode: t diff --git a/src/xnet/xnet_end_to_end_init.c b/src/xnet/xnet_end_to_end_init.c index c1101f98..886d551f 100644 --- a/src/xnet/xnet_end_to_end_init.c +++ b/src/xnet/xnet_end_to_end_init.c @@ -134,7 +134,7 @@ xint_e2e_worker_init(worker_data_t *wdp) { tdp = wdp->wd_tdp; wdp->wd_e2ep->e2e_sr_time = 0; - if(wdp->wd_e2ep->e2e_dest_hostname == NULL) { + if(xint_e2e_worker_dest_hostname(wdp) == NULL) { fprintf(xgp->errout,"%s: xint_e2e_worker_init: Target %d Worker Thread %d: No DESTINATION host name or IP address specified for this end-to-end operation.\n", xgp->progname, tdp->td_target_number, @@ -147,10 +147,10 @@ xint_e2e_worker_init(worker_data_t *wdp) { } // Get the IP address of the destination host - status = xint_lookup_addr(wdp->wd_e2ep->e2e_dest_hostname, 0, &addr); + status = xint_lookup_addr(xint_e2e_worker_dest_hostname(wdp), 0, &addr); if (status) { fprintf(xgp->errout, "%s: xint_e2e_worker_init: unable to identify host '%s'\n", - xgp->progname, wdp->wd_e2ep->e2e_dest_hostname); + xgp->progname, xint_e2e_worker_dest_hostname(wdp)); return(-1); } From 51ab516a8c9420701d84ba19e7a79ad398104bd1 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Fri, 25 Jul 2014 21:55:54 -0400 Subject: [PATCH 44/50] Add the last changes to implement multinic --- src/xnet/xnet_end_to_end.c | 39 ++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/src/xnet/xnet_end_to_end.c b/src/xnet/xnet_end_to_end.c index 5c027e64..76ba2183 100644 --- a/src/xnet/xnet_end_to_end.c +++ b/src/xnet/xnet_end_to_end.c @@ -63,16 +63,15 @@ do_connect(worker_data_t *wdp, int isdest) } int rc = 0; - xdd_e2e_ate_t *ate = NULL; // current address table entry target_data_t * const tdp = wdp->wd_tdp; - // Find an available address - for (ate = tdp->td_e2ep->e2e_address_table; 0 == ate->port_count; ate++); + // Get this worker's assigned address entry + xdd_e2e_ate_t *ate = worker_address_table_entry(wdp); // Resolve name to an IP address - rc = xint_lookup_addr(ate->hostname, 0, &tdp->td_e2ep->e2e_dest_addr); + rc = xint_lookup_addr(ate->hostname, 0, &wdp->wd_e2ep->e2e_dest_addr); assert(0 == rc); - struct in_addr addr = { .s_addr = tdp->td_e2ep->e2e_dest_addr }; + struct in_addr addr = { .s_addr = wdp->wd_e2ep->e2e_dest_addr }; char* ip_string = inet_ntoa(addr); if (!isdest) { @@ -87,10 +86,20 @@ do_connect(worker_data_t *wdp, int isdest) xni_bufset_t bufset; memset(&bufset, 0, sizeof(bufset)); bufset.bufs = tdp->io_buffers; - bufset.bufcount = tdp->io_buffers_count; + // Find the first buffer to be owned by this worker's connection + for (const xdd_e2e_ate_t *p = tdp->td_e2ep->e2e_address_table; + p != ate; + p++) { + + bufset.bufs += p->port_count; + } + bufset.bufcount = ate->port_count; // one buffer per port bufset.bufsize = tdp->io_buffer_size; bufset.reserved = getpagesize(); + // Check for overflow + assert(bufset.bufs+bufset.bufcount <= tdp->io_buffers+tdp->io_buffers_count); + if (isdest) { rc = xni_accept_connection(tdp->xni_ctx, &xep, &bufset, conn); } else { @@ -314,11 +323,10 @@ xint_is_e2e(const target_data_t *tdp) xni_connection_t* xint_e2e_worker_connection(worker_data_t *wdp) { - target_data_t * const tdp = wdp->wd_tdp; - - //TODO: support more than one host - assert(1 == tdp->td_e2ep->e2e_address_table_host_count); - xni_connection_t * const conn = tdp->td_e2ep->xni_td_connections; + const int idx = wdp->wd_e2ep->address_table_index; + xni_connection_t * const conn = idx >= 0 + ? wdp->wd_tdp->td_e2ep->xni_td_connections+idx + : NULL; return conn; } @@ -326,11 +334,10 @@ xint_e2e_worker_connection(worker_data_t *wdp) pthread_mutex_t* xint_e2e_worker_connection_mutex(worker_data_t *wdp) { - xint_e2e_t * const e2ep = wdp->wd_tdp->td_e2ep; - - //TODO: support more than one host - assert(1 == e2ep->e2e_address_table_host_count); - pthread_mutex_t * const mutex = e2ep->xni_td_connection_mutexes; + const int idx = wdp->wd_e2ep->address_table_index; + pthread_mutex_t * const mutex = idx >= 0 + ? wdp->wd_tdp->td_e2ep->xni_td_connection_mutexes+idx + : NULL; return mutex; } From 7579ec4aa37a0920aa35a6a40988856be33a0cc5 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Mon, 11 Aug 2014 21:29:32 -0400 Subject: [PATCH 45/50] Remove duplicate barrier implementation --- src/compat/xint_barrier.h | 93 --------------------------------------- src/compat/xint_darwin.h | 1 - 2 files changed, 94 deletions(-) delete mode 100644 src/compat/xint_barrier.h diff --git a/src/compat/xint_barrier.h b/src/compat/xint_barrier.h deleted file mode 100644 index d5c4a20f..00000000 --- a/src/compat/xint_barrier.h +++ /dev/null @@ -1,93 +0,0 @@ -/* - * XDD - a data movement and benchmarking toolkit - * - * Copyright (C) 1992-2013 I/O Performance, Inc. - * Copyright (C) 1999-2006 Brian Paul - * Copyright (C) 2008 Tungsten Graphics, Inc., Cedar Park, Texas. - * Copyright (C) 2009-2013 UT-Battelle, LLC - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License version 2, as published by the Free Software - * Foundation. See file COPYING. - * - */ -/* Pthread-based barrier adapted from Gallium */ -#ifndef XINT_BARRIER_H -#define XINT_BARRIER_H - -#include -#include -#include - -#define HAVE_XINT_BARRIER - -typedef struct xint_barrier { - size_t count; - size_t waiters; - pthread_mutex_t mutex; - pthread_cond_t cond; -} xint_barrier_t; - -inline static int xint_barrier_init(xint_barrier_t *barrier, size_t count) -{ - int rc = 0; - barrier->count = count; - barrier->waiters = 0; - rc = pthread_mutex_init(&barrier->mutex, NULL); - assert(0 == rc); - rc = pthread_cond_init(&barrier->cond, NULL); - assert(0 == rc); - return rc; -} - -inline static int xint_barrier_destroy(xint_barrier_t *barrier) -{ - //int rc = 0; - - // Have to remove the checking because the qthreads are sitting in a mutex - // even during a successful completion - //assert(barrier->waiters == 0); - pthread_cond_destroy(&barrier->cond); - //if (0 != rc) -// fprintf(stderr, -// "Error: Destroying barrier condvar count: %zd reason: %s\n", -// barrier->count, strerror(rc)); - //assert(0 == rc); - pthread_mutex_destroy(&barrier->mutex); -// if (0 != rc) -// fprintf(stderr, -// "Error: Destroying barrier mutex count: %zd reason: %s\n", -// barrier->count, strerror(rc)); - //assert(0 == rc); - return 0; -} - -inline static int xint_barrier_wait(xint_barrier_t *barrier) -{ - assert(barrier->waiters < barrier->count); - pthread_mutex_lock(&barrier->mutex); - barrier->waiters++; - - if (barrier->waiters == barrier->count) { - barrier->waiters = 0; - pthread_cond_broadcast(&barrier->cond); - } else { - pthread_cond_wait(&barrier->cond, &barrier->mutex); - } - - pthread_mutex_unlock(&barrier->mutex); - return 0; -} - -#endif -/* - * Local variables: - * indent-tabs-mode: t - * default-tab-width: 4 - * c-indent-level: 4 - * c-basic-offset: 4 - * End: - * - * vim: ts=4 sts=4 sw=4 noexpandtab - */ diff --git a/src/compat/xint_darwin.h b/src/compat/xint_darwin.h index 1a883790..fffaaa50 100644 --- a/src/compat/xint_darwin.h +++ b/src/compat/xint_darwin.h @@ -53,7 +53,6 @@ #endif /* XDD internal compatibility headers for this platform */ -#include "xint_barrier.h" #include "xint_nclk.h" /* nclk_t, prototype compatibility */ #include "xint_misc.h" From 6ab7b8f943754eea7d021b037584f84277e510fc Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Tue, 12 Aug 2014 17:24:37 -0400 Subject: [PATCH 46/50] Remove unused barriers --- src/base/target_init.c | 10 ---------- src/common/debug.c | 2 -- src/common/xint_td.h | 3 --- 3 files changed, 15 deletions(-) diff --git a/src/base/target_init.c b/src/base/target_init.c index 9610a5f2..43070ce1 100644 --- a/src/base/target_init.c +++ b/src/base/target_init.c @@ -203,16 +203,6 @@ xint_target_init_barriers(target_data_t *tdp) { sprintf(tmpname,"T%04d:target_worker_thread_init_barrier",tdp->td_target_number); status += xdd_init_barrier(tdp->td_planp, &tdp->td_target_worker_thread_init_barrier,2, tmpname); - // The Target Pass barrier - sprintf(tmpname,"T%04d>targetpass_worker_thread_passcomplete_barrier",tdp->td_target_number); - status += xdd_init_barrier(tdp->td_planp, &tdp->td_targetpass_worker_thread_passcomplete_barrier,tdp->td_queue_depth+1,tmpname); - - // The Target Pass E2E EOF Complete barrier - only initialized when an End-to-End operation is running - if (xint_is_e2e(tdp)) { - sprintf(tmpname,"T%04d>targetpass_worker_thread_eofcomplete_barrier",tdp->td_target_number); - status += xdd_init_barrier(tdp->td_planp, &tdp->td_targetpass_worker_thread_eofcomplete_barrier,2,tmpname); - } - // The Target Start Trigger barrier if (tdp->td_target_options & TO_WAITFORSTART) { // If we are expecting a Start Trigger then we need to init the starttrigger barrier sprintf(tmpname,"T%04d>target_target_starttrigger_barrier",tdp->td_target_number); diff --git a/src/common/debug.c b/src/common/debug.c index ee91ed3d..f8380f54 100644 --- a/src/common/debug.c +++ b/src/common/debug.c @@ -117,8 +117,6 @@ xdd_show_target_data(target_data_t *tdp) { fprintf(stderr,"xdd_show_target_data: char td_occupant_name[XDD_BARRIER_MAX_NAME_LENGTH]='%s'\n",tdp->td_occupant_name); // For a Target thread this is "TARGET####", for a Worker Thread it is "TARGET####WORKER####" fprintf(stderr,"xdd_show_target_data: xdd_barrier_t *td_current_barrier=%p\n",tdp->td_current_barrier); // Pointer to the current barrier this Thread is in at any given time or NULL if not in a barrier fprintf(stderr,"xdd_show_target_data: xdd_barrier_t td_target_worker_thread_init_barrier\n"); // Where the Target Thread waits for the Worker Thread to initialize - fprintf(stderr,"xdd_show_target_data: xdd_barrier_t td_targetpass_worker_thread_passcomplete_barrier\n");// The barrier used to sync targetpass() with all the Worker Threads at the end of a pass - fprintf(stderr,"xdd_show_target_data: xdd_barrier_t td_targetpass_worker_thread_eofcomplete_barrier\n"); // The barrier used to sync targetpass_eof_desintation_side() with a Worker Thread trying to recv an EOF packet fprintf(stderr,"xdd_show_target_data: uint64_t td_current_bytes_issued=%lld\n",(long long int)tdp->td_current_bytes_issued); // The amount of data for all transfer requests that has been issued so far fprintf(stderr,"xdd_show_target_data: uint64_t td_current_bytes_completed=%lld\n",(long long int)tdp->td_current_bytes_completed); // The amount of data for all transfer requests that has been completed so far fprintf(stderr,"xdd_show_target_data: uint64_t td_current_bytes_remaining=%lld\n",(long long int)tdp->td_current_bytes_remaining); // Bytes remaining to be transferred diff --git a/src/common/xint_td.h b/src/common/xint_td.h index 44353d27..b5dcc960 100644 --- a/src/common/xint_td.h +++ b/src/common/xint_td.h @@ -74,9 +74,6 @@ struct xint_target_data { // Target-specific variables xdd_barrier_t td_target_worker_thread_init_barrier; // Where the Target Thread waits for the Worker Thread to initialize - xdd_barrier_t td_targetpass_worker_thread_passcomplete_barrier;// The barrier used to sync targetpass() with all the Worker Threads at the end of a pass - xdd_barrier_t td_targetpass_worker_thread_eofcomplete_barrier;// The barrier used to sync targetpass_eof_desintation_side() with a Worker Thread trying to recv an EOF packet - uint64_t td_current_bytes_issued; // The amount of data for all transfer requests that has been issued so far uint64_t td_current_bytes_completed; // The amount of data for all transfer requests that has been completed so far From 1b879b480de40b398d83ca939be19f3b5b38cbbb Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Tue, 12 Aug 2014 17:58:50 -0400 Subject: [PATCH 47/50] Wait for worker threads to finish connecting A target thread should wait for all of its workers to finish connecting before returning from its init function. Synchronization is accomplished with a new barrier, td_target_worker_thread_connected_barrier. --- src/base/target_init.c | 19 +++++++++++++++++++ src/base/worker_thread.c | 14 ++++++++++---- src/common/xint_td.h | 1 + 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/base/target_init.c b/src/base/target_init.c index 43070ce1..a0367afd 100644 --- a/src/base/target_init.c +++ b/src/base/target_init.c @@ -209,6 +209,18 @@ xint_target_init_barriers(target_data_t *tdp) { status += xdd_init_barrier(tdp->td_planp, &tdp->td_trigp->target_target_starttrigger_barrier,2,tmpname); } + // Barrier to wait for all worker threads to connect (E2E only) + if (xint_is_e2e(tdp)) { + snprintf(tmpname, + sizeof(tmpname), + "T%04d>target_worker_thread_connected_barrier", + tdp->td_target_number); + status += xdd_init_barrier(tdp->td_planp, + &tdp->td_target_worker_thread_connected_barrier, + (tdp->td_queue_depth + 1), // workers + target + tmpname); + } + // The "td_counters_mutex" is used by the WorkerThreads when updating the counter information in the Target Thread Data status += pthread_mutex_init(&tdp->td_counters_mutex, 0); @@ -335,6 +347,13 @@ xint_target_init_start_worker_threads(target_data_t *tdp) { wdp = wdp->wd_next_wdp; } // End of FOR loop that starts all worker_threads for this target + // Wait for worker threads to finish connecting (E2E only) + if (xint_is_e2e(tdp)) { + xdd_barrier(&tdp->td_target_worker_thread_connected_barrier, + &tdp->td_occupant, + 1); + } + if (xgp->global_options & GO_REALLYVERBOSE) { fprintf(xgp->errout,"\n%s: xdd_target_init_start_worker_threads: Target %d ALL %d WorkerThreads started\n", xgp->progname, diff --git a/src/base/worker_thread.c b/src/base/worker_thread.c index 31bc01cb..7e0083f8 100644 --- a/src/base/worker_thread.c +++ b/src/base/worker_thread.c @@ -45,9 +45,7 @@ xdd_worker_thread(void *pin) { // Enter the WorkerThread_Init barrier so that the next WorkerThread can start xdd_barrier(&tdp->td_target_worker_thread_init_barrier,&wdp->wd_occupant,0); - if ( xgp->abort == 1) // Something went wrong during thread initialization so let's just leave - return(0); - + // Only for E2E if (xint_is_e2e(tdp)) { // Set up for an e2e operation and establish a connection status = xint_e2e_worker_init(wdp); @@ -58,10 +56,18 @@ xdd_worker_thread(void *pin) { tdp->td_target_number, wdp->wd_worker_number, (tdp->td_target_options & TO_E2E_DESTINATION) ? "DESTINATION":"SOURCE"); - return(0); + xgp->abort = 1; } + + // Enter barrier to let the target thread know we have connected + xdd_barrier(&tdp->td_target_worker_thread_connected_barrier, + &wdp->wd_occupant, + 0); } + if (1 == xgp->abort) // Something went wrong during thread initialization so let's just leave + return(0); + // Set the buffer data pattern for non-E2E operations or E2E sources if (!xint_is_e2e(tdp) || !(tdp->td_target_options & TO_E2E_DESTINATION)) { xdd_datapattern_buffer_init(wdp); diff --git a/src/common/xint_td.h b/src/common/xint_td.h index b5dcc960..9487f9a4 100644 --- a/src/common/xint_td.h +++ b/src/common/xint_td.h @@ -73,6 +73,7 @@ struct xint_target_data { // Target-specific variables xdd_barrier_t td_target_worker_thread_init_barrier; // Where the Target Thread waits for the Worker Thread to initialize + xdd_barrier_t td_target_worker_thread_connected_barrier; // Where the Target Thread waits for every Worker Thread to establish a connection (End-to-End only) uint64_t td_current_bytes_issued; // The amount of data for all transfer requests that has been issued so far From 492e57b347fd878474eef25e37bebca6022d273c Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Mon, 18 Aug 2014 05:15:22 -0400 Subject: [PATCH 48/50] Automatically generate config.h.in After cleaning up some templates it's now possible (and highly recommended) to use autoheader(1) to generate config.h.in instead of the old method of editing it by hand. The file config checks were modified to stop generating preprocessor symbols. --- Makefile.in | 3 +- README | 1 + configure.ac | 39 +++-- contrib/buildbot_gen_test_config.sh | 4 +- doc/HOWTO_release.txt | 2 + src/compat/config.h.in | 262 +++++++++++++--------------- 6 files changed, 148 insertions(+), 163 deletions(-) diff --git a/Makefile.in b/Makefile.in index 8621a347..b696ae12 100644 --- a/Makefile.in +++ b/Makefile.in @@ -77,6 +77,7 @@ XNI_DIR = $(SRC_DIR)/xni # AR = ar AUTOCONF = autoconf +AUTOHEADER = autoheader CC = $(CC_EXE) CP = cp CPR = cp -r $(CP_PRESERVE_OPTS) @@ -422,7 +423,7 @@ CONFIG_FLAGS = $(shell grep "\-\-prefix" config.log | cut -f 5- -d ' ') xdd-$(XDDVERSION): git archive --format=tar --prefix=$@/ master | $(TAR) xf - $(FIND) $@ -name .gitignore -exec rm {} \; - cd $@ && $(AUTOCONF) + cd $@ && $(AUTOCONF) && $(AUTOHEADER) xdd-$(XDDVERSION).tar.gz: xdd-$(XDDVERSION) $(TAR) cfz $@ $< diff --git a/README b/README index 2d0359ac..99540014 100644 --- a/README +++ b/README @@ -23,6 +23,7 @@ software build process. We also suggest enabling debug mode. On supported OS: > autoconf +> autoheader > ./configure --enable-debug > make > make install diff --git a/configure.ac b/configure.ac index 66869d36..0ff8d4ed 100644 --- a/configure.ac +++ b/configure.ac @@ -24,8 +24,7 @@ dnl Process this file with autoconf to produce a configure script dnl AC_PREREQ(2.59) AC_INIT([xdd], [7.0.0.pre-rc27], bug@xdd.org) -AC_CONFIG_HEADER([src/compat/config.h]) - +AC_CONFIG_HEADERS([src/compat/config.h]) dnl dnl Include useful macros @@ -282,13 +281,15 @@ AC_ARG_ENABLE([numa], [enable_numa="$enableval"], [enable_numa=yes]) if test "$enable_numa" = "yes" ; then - AC_DEFINE(HAVE_ENABLE_NUMA) + AC_DEFINE(HAVE_ENABLE_NUMA,1,[Define to 1 if you have libnuma.]) AC_SEARCH_LIBS([numa_node_to_cpus], [numa], - AC_DEFINE(HAVE_NUMA_NODE_TO_CPUS), + AC_DEFINE(HAVE_NUMA_NODE_TO_CPUS,1, + [Define to 1 if you have the `numa_node_to_cpus' function.]), AC_MSG_ERROR([Function numa_node_to_cpus not found. Use --disable-numa])) AC_SEARCH_LIBS([numa_allocate_cpumask], [numa], - AC_DEFINE(HAVE_NUMA_ALLOCATE_CPUMASK), + AC_DEFINE(HAVE_NUMA_ALLOCATE_CPUMASK,1, + [Define to 1 if you have the `numa_allocate_cpumask' function.]), AC_MSG_ERROR([Function numa_allocate_cpumask not found. Use --disable-numa.])) fi @@ -315,10 +316,12 @@ AC_ARG_ENABLE([ib], [enable_ib="$enableval"], [enable_ib=yes]) if test "x$enable_ib" = "xyes" ; then - AC_DEFINE(HAVE_ENABLE_IB) + AC_DEFINE(HAVE_ENABLE_IB,1, + [Define to 1 to enable InfiniBand network support.]) AC_CHECK_HEADERS([infiniband/verbs.h]) AC_SEARCH_LIBS([ibv_get_device_list], [ibverbs], - AC_DEFINE(HAVE_IBV_GET_DEVICE_LIST), + AC_DEFINE(HAVE_IBV_GET_DEVICE_LIST,1, + [Define to 1 if you have the `ibv_get_device_list' function.]), AC_MSG_ERROR([Function ibv_get_device_list not found. Use --disable-ib.])) fi @@ -336,7 +339,8 @@ AC_ARG_ENABLE([xfs], [enable_xfs=yes]) if test "$enable_xfs" = "yes" ; then - AC_DEFINE(HAVE_ENABLE_XFS) + AC_DEFINE(HAVE_ENABLE_XFS,1, + [Define to 1 to enable XFS preallocation support.]) xfs_header="no" AC_CHECK_HEADERS([xfs/xfs.h] [xfs/libxfs.h], xfs_header="yes"; break, []) @@ -344,7 +348,8 @@ if test "$enable_xfs" = "yes" ; then AC_MSG_ERROR([No valid XFS header found (sometimes this is caused by missing libuuid headers). Use --disable-xfs.]) fi AC_CHECK_DECLS([xfsctl], - AC_DEFINE(HAVE_XFSCTL), + AC_DEFINE(HAVE_XFSCTL,1, + [Define to 1 if you have the `xfsctl' function.]), AC_MSG_ERROR([Function xfsctl not found. Use --disable-xfs.]), [#if HAVE_XFS_XFS_H #include @@ -431,14 +436,14 @@ AX_PYTHON() dnl dnl Check for export-controlled packages dnl -AC_CHECK_FILES([contrib/pycrypto-2.6.1.tar.gz], - AC_SUBST(PYCRYPTO_DIST, [contrib/pycrypto-2.6.1.tar.gz]), - [AX_PYTHON_MODULE([Crypto], - [XDD requires PyCrypto. Due to U.S. Export Controls, PyCrypto must be installed separately from XDD. See README.crypto])]) -AC_CHECK_FILES([contrib/ecdsa-0.11.tar.gz], - AC_SUBST(ECDSA_DIST, [contrib/ecdsa-0.11.tar.gz]), - [AX_PYTHON_MODULE([ecdsa], - [XDD requires Python ECDSA. Due to U.S. Export Controls, ECDSA must be installed separately from XDD. See README.crypto])]) +AC_CHECK_FILE([contrib/pycrypto-2.6.1.tar.gz], + AC_SUBST(PYCRYPTO_DIST, [contrib/pycrypto-2.6.1.tar.gz]), + [AX_PYTHON_MODULE([Crypto], + [XDD requires PyCrypto. Due to U.S. Export Controls, PyCrypto must be installed separately from XDD. See README.crypto])]) +AC_CHECK_FILE([contrib/ecdsa-0.11.tar.gz], + AC_SUBST(ECDSA_DIST, [contrib/ecdsa-0.11.tar.gz]), + [AX_PYTHON_MODULE([ecdsa], + [XDD requires Python ECDSA. Due to U.S. Export Controls, ECDSA must be installed separately from XDD. See README.crypto])]) dnl dnl Check for packages XDD requires directly diff --git a/contrib/buildbot_gen_test_config.sh b/contrib/buildbot_gen_test_config.sh index e2da2ff6..57b17fdd 100755 --- a/contrib/buildbot_gen_test_config.sh +++ b/contrib/buildbot_gen_test_config.sh @@ -4,7 +4,8 @@ # # Step 1: Generate a local test_config # Step 2: autoconf -# Step 3: ./configure +# Step 3: autoheader +# Step 4: ./configure # # @@ -128,4 +129,5 @@ EOF # Perform the configure # autoconf +autoheader ./configure $configure_flags diff --git a/doc/HOWTO_release.txt b/doc/HOWTO_release.txt index 33f2b315..b16d3ee4 100644 --- a/doc/HOWTO_release.txt +++ b/doc/HOWTO_release.txt @@ -19,6 +19,7 @@ available: > cd xdd > vi configure.ac # and edit the VERSION variable > autoconf + > autoheader > ./configure --prefix=$HOME/sw/xdd > make baseversion > make @@ -55,5 +56,6 @@ The resulting tar file now contains the released code only. 9. Prepare for the next release > vi configure.ac # And set the version in AC_INIT to pre-nextrelease > autoconf + > autoheader > git commit -a > git push diff --git a/src/compat/config.h.in b/src/compat/config.h.in index 69302170..fe1bba93 100644 --- a/src/compat/config.h.in +++ b/src/compat/config.h.in @@ -1,50 +1,60 @@ -/* - * XDD - a data movement and benchmarking toolkit - * - * Copyright (C) 1992-2013 I/O Performance, Inc. - * Copyright (C) 2009-2013 UT-Battelle, LLC - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public - * License version 2, as published by the Free Software - * Foundation. See file COPYING. - * - */ - -#ifndef CONFIG_H -#define CONFIG_H - -/******************************************************************************** - * Autoconf stuff - *******************************************************************************/ +/* src/compat/config.h.in. Generated from configure.ac by autoheader. */ -/* Define to the address where bug reports for this package should be sent. */ -#undef PACKAGE_BUGREPORT +/* Define if you have the cpu_set_t type. */ +#undef HAVE_CPU_SET_T -/* Define to the full name of this package. */ -#undef PACKAGE_NAME +/* Define to 1 if you have the declaration of `BLKGETSIZE64', and to 0 if you + don't. */ +#undef HAVE_DECL_BLKGETSIZE64 -/* Define to the full name and version of this package. */ -#undef PACKAGE_STRING +/* Define to 1 if you have the declaration of `DKIOCGETBLOCKCOUNT', and to 0 + if you don't. */ +#undef HAVE_DECL_DKIOCGETBLOCKCOUNT -/* Define to the one symbol short name of this package. */ -#undef PACKAGE_TARNAME +/* Define to 1 if you have the declaration of `DKIOCGETBLOCKSIZE', and to 0 if + you don't. */ +#undef HAVE_DECL_DKIOCGETBLOCKSIZE -/* Define to the home page for this package. */ -#undef PACKAGE_URL +/* Define to 1 if you have the declaration of `TCP_CONGESTION', and to 0 if + you don't. */ +#undef HAVE_DECL_TCP_CONGESTION -/* Define to the version of this package. */ -#undef PACKAGE_VERSION +/* Define to 1 if you have the declaration of `xfsctl', and to 0 if you don't. + */ +#undef HAVE_DECL_XFSCTL +/* Define to 1 if you have the declaration of `XFS_SUPER_MAGIC', and to 0 if + you don't. */ +#undef HAVE_DECL_XFS_SUPER_MAGIC -/******************************************************************************** - * Defines for headers - *******************************************************************************/ +/* Define to 1 to enable InfiniBand network support. */ +#undef HAVE_ENABLE_IB + +/* Define to 1 if you have libnuma. */ +#undef HAVE_ENABLE_NUMA + +/* Define to 1 to enable XFS preallocation support. */ +#undef HAVE_ENABLE_XFS + +/* Define to 1 if you have the header file. */ +#undef HAVE_GTEST_GTEST_H + +/* Define to 1 if you have the `ibv_get_device_list' function. */ +#undef HAVE_IBV_GET_DEVICE_LIST + +/* Define to 1 if you have the header file. */ +#undef HAVE_INFINIBAND_VERBS_H + +/* Define to 1 if you have the `initstate' function. */ +#undef HAVE_INITSTATE /* Define to 1 if you have the header file. */ #undef HAVE_INTTYPES_H -/* Define to 1 if you have the header file. */ +/* Define to 1 if you have the `ioctl' function. */ +#undef HAVE_IOCTL + +/* Define to 1 if you have the header file. */ #undef HAVE_LIBGEN_H /* Define to 1 if you have the header file. */ @@ -52,6 +62,58 @@ /* Define to 1 if you have the header file. */ #undef HAVE_MEMORY_H + +/* Define to 1 if you have the `memset' function. */ +#undef HAVE_MEMSET + +/* Define if you have the MPI library. */ +#undef HAVE_MPI + +/* Define to 1 if you have the `nanosleep' function. */ +#undef HAVE_NANOSLEEP + +/* Define to 1 if you have the `numa_allocate_cpumask' function. */ +#undef HAVE_NUMA_ALLOCATE_CPUMASK + +/* Define to 1 if you have the header file. */ +#undef HAVE_NUMA_H + +/* Define to 1 if you have the `numa_node_to_cpus' function. */ +#undef HAVE_NUMA_NODE_TO_CPUS + +/* Define to 1 if you have the `posix_memalign' function. */ +#undef HAVE_POSIX_MEMALIGN + +/* Define to 1 if you have the `pread' function. */ +#undef HAVE_PREAD + +/* Define if you have pthread_ */ +#undef HAVE_PTHREAD_ATTR_SETAFFINITY_NP + +/* Define if you have pthread_barrier_t type. */ +#undef HAVE_PTHREAD_BARRIER_T + +/* Define to 1 if you have the `pwrite' function. */ +#undef HAVE_PWRITE + +/* Define to 1 if you have the `rand' function. */ +#undef HAVE_RAND + +/* Define to 1 if you have the `random' function. */ +#undef HAVE_RANDOM + +/* Define to 1 if you have the `sched_getcpu' function. */ +#undef HAVE_SCHED_GETCPU + +/* Define to 1 if you have the header file. */ +#undef HAVE_SCHED_H + +/* Define to 1 if you have the `sched_setscheduler' function. */ +#undef HAVE_SCHED_SETSCHEDULER + +/* Define to 1 if you have the header file. */ +#undef HAVE_SCSI_SG_H + /* Define to 1 if you have the header file. */ #undef HAVE_STDINT_H @@ -67,6 +129,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_SYS_DISK_H +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_IOCTL_H + /* Define to 1 if you have the header file. */ #undef HAVE_SYS_MOUNT_H @@ -82,129 +147,38 @@ /* Define to 1 if you have the header file. */ #undef HAVE_UTMPX_H +/* Define to 1 if you have the `valloc' function. */ +#undef HAVE_VALLOC + +/* Define to 1 if you have the `xfsctl' function. */ +#undef HAVE_XFSCTL + /* Define to 1 if you have the header file. */ #undef HAVE_XFS_LIBXFS_H /* Define to 1 if you have the header file. */ #undef HAVE_XFS_XFS_H -/* Define to 1 if you have the header file. */ -#undef HAVE_NUMA_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_SCHED_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_SCSI_SG_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_GTEST_GTEST_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_INFINIBAND_VERBS_H - -/******************************************************************************** - * Defines for functions - *******************************************************************************/ - -/* Define to 1 if you have the `clock_gettime' function. */ -#undef HAVE_CLOCK_GETTIME - -/* Define to 1 if you have the `ibv_get_device_list' function. */ -#undef HAVE_IBV_GET_DEVICE_LIST - -/* Define to 1 if you have the `initstate' function. */ -#undef HAVE_INITSTATE - -/* Define to 1 if you have the `memset' function. */ -#undef HAVE_MEMSET - -/* Define to 1 if you have the `nanosleep' function. */ -#undef HAVE_NANOSLEEP - -/* Define to 1 if you have the `posix_memalign' function. */ -#undef HAVE_NUMA_NODE_TO_CPUS - -/* Define to 1 if you have the `posix_memalign' function. */ -#undef HAVE_NUMA_ALLOCATE_CPUMASK - -/* Define to 1 if you have the `posix_memalign' function. */ -#undef HAVE_POSIX_MEMALIGN - -/* Define to 1 if you have the `pread' function. */ -#undef HAVE_PREAD - -/* Define to 1 if you have the `pthread_attr_setaffinity_np' function. */ -#undef HAVE_PTHREAD_ATTR_SETAFFINITY_NP - -/* Define to 1 if you have the `pwrite' function. */ -#undef HAVE_PWRITE - -/* Define to 1 if you have the `random' function. */ -#undef HAVE_RANDOM - -/* Define to 1 if you have the `rand' function. */ -#undef HAVE_RAND - -/* Define to 1 if you have the `sched_getcpu' function. */ -#undef HAVE_SCHED_GETCPU - -/* Define to 1 if you have the `sched_setscheduler' function. */ -#undef HAVE_SCHED_SETSCHEDULER - -/******************************************************************************** - * Defines for features - *******************************************************************************/ - -/* Define if you have BLKGETSIZE64 ioctl */ -#undef HAVE_DECL_BLKGETSIZE64 - -/* Define if you have cpu_set_t type */ -#undef HAVE_CPU_SET_T - -/* Define if you have BLKGETSIZE64 ioctl */ -#undef HAVE_DECL_DKIOCGETBLOCKCOUNT +/* Define to the address where bug reports for this package should be sent. */ +#undef PACKAGE_BUGREPORT -/* Define if you have BLKGETSIZE64 ioctl */ -#undef HAVE_DECL_DKIOCGETBLOCKSIZE +/* Define to the full name of this package. */ +#undef PACKAGE_NAME -/* Define to 1 if you have the declaration of `XFS_SUPER_MAGIC', and to 0 if - you don't. */ -#undef HAVE_DECL_XFS_SUPER_MAGIC +/* Define to the full name and version of this package. */ +#undef PACKAGE_STRING -/* Define to 1 if you have the declaration of xfsctl, and to 0 if you don't. */ -#undef HAVE_DECL_XFSCTL +/* Define to the one symbol short name of this package. */ +#undef PACKAGE_TARNAME -/* Define if you have the MPI library. */ -#undef HAVE_MPI +/* Define to the home page for this package. */ +#undef PACKAGE_URL -/* Define if you have the pthread. */ -#undef HAVE_PTHREAD_BARRIER_T +/* Define to the version of this package. */ +#undef PACKAGE_VERSION /* Define to 1 if you have the ANSI C header files. */ #undef STDC_HEADERS /* Define to `unsigned int' if does not define. */ #undef size_t - -/* Define to 1 if you wish to enable XFS support */ -#undef HAVE_ENABLE_XFS - -/* Define to 1 if you wish to enable Infiniband support */ -#undef HAVE_ENABLE_IB - -/* Define to 1 if the TCP_CONGESTION setsockopt is available */ -#undef HAVE_DECL_TCP_CONGESTION - - -#endif - -/* - * Local variables: - * indent-tabs-mode: t - * c-indent-level: 4 - * c-basic-offset: 4 - * End: - * - * vim: ts=4 sts=4 sw=4 noexpandtab - */ From 6603c10dc315feb6d4e9e986137c8f93dcd7da34 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Mon, 18 Aug 2014 06:16:04 -0400 Subject: [PATCH 49/50] Fix bug in error handling during TOT initialization This bug was exposed thanks to the new build system actually defining HAVE_VALLOC. --- src/base/target_offset_table.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/base/target_offset_table.c b/src/base/target_offset_table.c index 37242ef2..c668280a 100644 --- a/src/base/target_offset_table.c +++ b/src/base/target_offset_table.c @@ -49,14 +49,14 @@ int tot_init(tot_t** table, size_t queue_depth, size_t num_reqs) // Initialize the memory in the dumbest way possible #if HAVE_VALLOC *table = valloc(sizeof(**table) + num_entries * sizeof(tot_entry_t)); - rc = (NULL != table); + rc = (NULL == *table); #elif HAVE_POSIX_MEMALIGN rc = posix_memalign((void**)table, sysconf(_SC_PAGESIZE), sizeof(**table) + num_entries * sizeof(tot_entry_t)); #else *table = malloc(sizeof(**table) + num_entries * sizeof(tot_entry_t)); - rc = (NULL != table); + rc = (NULL == *table); #endif if (0 != rc) return -1; From 77aa8fa5b9d0dd63627767742818f9197f9ce7d2 Mon Sep 17 00:00:00 2001 From: Nick Mills Date: Mon, 18 Aug 2014 14:57:24 -0400 Subject: [PATCH 50/50] Remove config.h.in from version control This file is automatically generated by autoheader(1). --- src/compat/config.h.in | 184 ----------------------------------------- 1 file changed, 184 deletions(-) delete mode 100644 src/compat/config.h.in diff --git a/src/compat/config.h.in b/src/compat/config.h.in deleted file mode 100644 index fe1bba93..00000000 --- a/src/compat/config.h.in +++ /dev/null @@ -1,184 +0,0 @@ -/* src/compat/config.h.in. Generated from configure.ac by autoheader. */ - -/* Define if you have the cpu_set_t type. */ -#undef HAVE_CPU_SET_T - -/* Define to 1 if you have the declaration of `BLKGETSIZE64', and to 0 if you - don't. */ -#undef HAVE_DECL_BLKGETSIZE64 - -/* Define to 1 if you have the declaration of `DKIOCGETBLOCKCOUNT', and to 0 - if you don't. */ -#undef HAVE_DECL_DKIOCGETBLOCKCOUNT - -/* Define to 1 if you have the declaration of `DKIOCGETBLOCKSIZE', and to 0 if - you don't. */ -#undef HAVE_DECL_DKIOCGETBLOCKSIZE - -/* Define to 1 if you have the declaration of `TCP_CONGESTION', and to 0 if - you don't. */ -#undef HAVE_DECL_TCP_CONGESTION - -/* Define to 1 if you have the declaration of `xfsctl', and to 0 if you don't. - */ -#undef HAVE_DECL_XFSCTL - -/* Define to 1 if you have the declaration of `XFS_SUPER_MAGIC', and to 0 if - you don't. */ -#undef HAVE_DECL_XFS_SUPER_MAGIC - -/* Define to 1 to enable InfiniBand network support. */ -#undef HAVE_ENABLE_IB - -/* Define to 1 if you have libnuma. */ -#undef HAVE_ENABLE_NUMA - -/* Define to 1 to enable XFS preallocation support. */ -#undef HAVE_ENABLE_XFS - -/* Define to 1 if you have the header file. */ -#undef HAVE_GTEST_GTEST_H - -/* Define to 1 if you have the `ibv_get_device_list' function. */ -#undef HAVE_IBV_GET_DEVICE_LIST - -/* Define to 1 if you have the header file. */ -#undef HAVE_INFINIBAND_VERBS_H - -/* Define to 1 if you have the `initstate' function. */ -#undef HAVE_INITSTATE - -/* Define to 1 if you have the header file. */ -#undef HAVE_INTTYPES_H - -/* Define to 1 if you have the `ioctl' function. */ -#undef HAVE_IOCTL - -/* Define to 1 if you have the header file. */ -#undef HAVE_LIBGEN_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_LINUX_MAGIC_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_MEMORY_H - -/* Define to 1 if you have the `memset' function. */ -#undef HAVE_MEMSET - -/* Define if you have the MPI library. */ -#undef HAVE_MPI - -/* Define to 1 if you have the `nanosleep' function. */ -#undef HAVE_NANOSLEEP - -/* Define to 1 if you have the `numa_allocate_cpumask' function. */ -#undef HAVE_NUMA_ALLOCATE_CPUMASK - -/* Define to 1 if you have the header file. */ -#undef HAVE_NUMA_H - -/* Define to 1 if you have the `numa_node_to_cpus' function. */ -#undef HAVE_NUMA_NODE_TO_CPUS - -/* Define to 1 if you have the `posix_memalign' function. */ -#undef HAVE_POSIX_MEMALIGN - -/* Define to 1 if you have the `pread' function. */ -#undef HAVE_PREAD - -/* Define if you have pthread_ */ -#undef HAVE_PTHREAD_ATTR_SETAFFINITY_NP - -/* Define if you have pthread_barrier_t type. */ -#undef HAVE_PTHREAD_BARRIER_T - -/* Define to 1 if you have the `pwrite' function. */ -#undef HAVE_PWRITE - -/* Define to 1 if you have the `rand' function. */ -#undef HAVE_RAND - -/* Define to 1 if you have the `random' function. */ -#undef HAVE_RANDOM - -/* Define to 1 if you have the `sched_getcpu' function. */ -#undef HAVE_SCHED_GETCPU - -/* Define to 1 if you have the header file. */ -#undef HAVE_SCHED_H - -/* Define to 1 if you have the `sched_setscheduler' function. */ -#undef HAVE_SCHED_SETSCHEDULER - -/* Define to 1 if you have the header file. */ -#undef HAVE_SCSI_SG_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_STDINT_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_STDLIB_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_STRINGS_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_STRING_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_DISK_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_IOCTL_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_MOUNT_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_STAT_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_SYS_TYPES_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_UNISTD_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_UTMPX_H - -/* Define to 1 if you have the `valloc' function. */ -#undef HAVE_VALLOC - -/* Define to 1 if you have the `xfsctl' function. */ -#undef HAVE_XFSCTL - -/* Define to 1 if you have the header file. */ -#undef HAVE_XFS_LIBXFS_H - -/* Define to 1 if you have the header file. */ -#undef HAVE_XFS_XFS_H - -/* Define to the address where bug reports for this package should be sent. */ -#undef PACKAGE_BUGREPORT - -/* Define to the full name of this package. */ -#undef PACKAGE_NAME - -/* Define to the full name and version of this package. */ -#undef PACKAGE_STRING - -/* Define to the one symbol short name of this package. */ -#undef PACKAGE_TARNAME - -/* Define to the home page for this package. */ -#undef PACKAGE_URL - -/* Define to the version of this package. */ -#undef PACKAGE_VERSION - -/* Define to 1 if you have the ANSI C header files. */ -#undef STDC_HEADERS - -/* Define to `unsigned int' if does not define. */ -#undef size_t