pax_global_header00006660000000000000000000000064126521576140014523gustar00rootroot0000000000000052 comment=1a86169d59cec933e7e8018c8f73e9c61eecc992 toro-1.0.1/000077500000000000000000000000001265215761400125055ustar00rootroot00000000000000toro-1.0.1/.gitignore000066400000000000000000000000051265215761400144700ustar00rootroot00000000000000*.pyctoro-1.0.1/.travis.yml000066400000000000000000000014111265215761400146130ustar00rootroot00000000000000language: python python: - "2.6" - "2.7" - "3.2" - "3.3" - "3.4" - "pypy" # Futures were introduced in Python 3.2 in the `concurrent.futures` package. # This package has also been backported to older versions of Python and can be # installed with `pip install futures`. Tornado uses concurrent.futures if # available; otherwise it will use a compatible class defined in # tornado.concurrent. We want to test all cases: with and without the 'futures' # package installed in Python < 3.2, as well as Python >= 3.2. env: - FUTURES=yes - FUTURES=no matrix: exclude: - python: "3.2" env: FUTURES=yes - python: "3.3" env: FUTURES=yes before_script: - if [[ "$FUTURES" == "yes" ]]; then pip install futures; fi script: "python setup.py test" toro-1.0.1/LICENSE000066400000000000000000000010661265215761400135150ustar00rootroot00000000000000Toro Copyright (c) 2012 A. Jesse Jiryu Davis Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. toro-1.0.1/MANIFEST.in000066400000000000000000000003071265215761400142430ustar00rootroot00000000000000include README.rst include LICENSE recursive-include doc *.rst recursive-include doc *.py recursive-include doc *.png recursive-include doc Makefile prune doc/_build recursive-include examples *.py toro-1.0.1/README.rst000066400000000000000000000037761265215761400142110ustar00rootroot00000000000000==== toro ==== .. image:: https://raw.github.com/ajdavis/toro/master/doc/_static/toro.png :Info: Synchronization primitives for Tornado coroutines. :Author: A\. Jesse Jiryu Davis Documentation: http://toro.readthedocs.org/ .. important:: Toro is completed and deprecated; its features have been merged into Tornado. Development of locks and queues for Tornado coroutines continues in Tornado itself. .. image:: https://travis-ci.org/ajdavis/toro.png :target: https://travis-ci.org/ajdavis/toro About ===== A set of locking and synchronizing primitives analogous to those in Python's `threading module`_ or Gevent's `coros`_, for use with Tornado's `gen.engine`_. .. _threading module: http://docs.python.org/library/threading.html .. _coros: http://www.gevent.org/gevent.coros.html .. _gen.engine: http://www.tornadoweb.org/documentation/gen.html Dependencies ============ Tornado_ >= version 3.0. .. _Tornado: http://www.tornadoweb.org/ Examples ======== Here's a basic example (for more see the *examples* section of the docs): .. code-block:: python from tornado import ioloop, gen import toro q = toro.JoinableQueue(maxsize=3) @gen.coroutine def consumer(): while True: item = yield q.get() try: print 'Doing work on', item finally: q.task_done() @gen.coroutine def producer(): for item in range(10): yield q.put(item) producer() consumer() loop = ioloop.IOLoop.instance() # block until all tasks are done q.join().add_done_callback(loop.stop) loop.start() Documentation ============= You will need Sphinx_ and GraphViz_ installed to generate the documentation. Documentation can be generated like: .. code-block:: console $ sphinx-build doc build .. _Sphinx: http://sphinx.pocoo.org/ .. _GraphViz: http://www.graphviz.org/ Testing ======= Run ``python setup.py test`` in the root directory. Toro boasts 100% code coverage, including branch-coverage! toro-1.0.1/doc/000077500000000000000000000000001265215761400132525ustar00rootroot00000000000000toro-1.0.1/doc/Makefile000066400000000000000000000126641265215761400147230ustar00rootroot00000000000000# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " singlehtml to make a single large HTML file" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " text to make text files" @echo " man to make manual pages" @echo " texinfo to make Texinfo files" @echo " info to make Texinfo files and run them through makeinfo" @echo " gettext to make PO message catalogs" @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: -rm -rf $(BUILDDIR)/* html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Toro.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Toro.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @echo "# mkdir -p $$HOME/.local/share/devhelp/Toro" @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Toro" @echo "# devhelp" epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." toro-1.0.1/doc/_static/000077500000000000000000000000001265215761400147005ustar00rootroot00000000000000toro-1.0.1/doc/_static/toro.png000066400000000000000000000752371265215761400164070ustar00rootroot00000000000000PNG  IHDR,"tEXtSoftwareAdobe ImageReadyqe<iTXtXML:com.adobe.xmp a PLTE$ܳ\jJCڃu'%-,b!1 &,A}}cD=㸫0."$ Ś]?9}vvwSKlccrOHhZ\{Ԙmukk磋֨aB;&.D]STM@AuQJeV?($2vhTD-*)K[Ϧ{P62~^f-^ǷH0,X<7ڭK=.1p oLE'gF@xTN(JYf'BSpzݮE9;hþ$)!PD:!hZJZM@&?L_FMr]p 3?0-_@:YE+ =#)`A:_@9_A:`@:aA:`@9^?:&>P󶱲hQ-"3=sPItQItPI5ء\8IDATx |T?>tRS:0&1R.2"҆mIf^P,:$ m(IKG-vRآKkief3Xz;<圙L@n39>xOkIǟ;pkuN+59xɀ5MKk5#i]:`im_ <7\J`Xco]B`iWkCӚ̈ ;5|:s=Wk$bK t#]𡃵+KhzYZ&`}wtY4Z` F`ڲK g]pvj6]:`mX3Kڵ4LP h3MڔKƏީ1g$*0ciO5څ_e\coOJy"X ݙVeӪ|x`v̓"eMTڞ`/n]LgX Hk?L]:}u` AB앗Xp^z~8`%X^r원J?FBҗXh_ypʕҊl0{ ֤K@Ow\2U._X>d:10o]q]|ao\BF"}g咕+N,"_?H;k.ļ+p=UM耍{uv4ar\,.6X śξ#}&F G?"z;73 qĉG l#tӃK`#/Xs>hᒩw 1(8c̭ "qҝFE?88RUE^9Ek/m1_siאI˪@u= ?cŦY+ Z?WXp&k|tEcD?XXDδخ_YТֵG)ro0We xNNg;G88LX0NxsTWǮ>X|O֤X|SP##/El纗QZc %8<wG?ӛ4D/m!`Tsv"Ν;k垄i;de5̫W`| Z2&")ޞn|kAMrX}ٴQ#FMvoI0i g~z91'^~ҒrVSoypkݤwJx, aXFkNtE29Kj;SaO/ph}c?ۦ''~s[%+ ]F@޼`Qmdњjb!S\(N%$S7o'X!4h=+o_{N}ʟn(+"5+ 0>v%7q}8L[ӆ#Ih 0)Fyc䕟,t|n߹O_穳g¦Z~-|g*O:)ր陆,˜kGJ@c7'_~RXM`5m]tOZ[f?8{\Som?SdCķ/XS?}O 1u|W}\;>;P.˿]3w;?-G?Z1s̙3+`k'AK\+ ]S`¿>I~| Zg?dB*qj\~_go}Jk]<9ah1}HsBBo!R~_ԶiT4Y)JaXbt$-Zr+Y"h[@=0n|/Z+SnOf 'I|Y-26>ΚC坏< ɒBzAH\3pZ+ff`?+Nd _nOq$V\6_wg}ϒjfI_j4 'H*"QuTs\pEX4s…Y"l2L:?Dj?;<ȝ}MA.pTx?ZAXa}#i$4Sdnwә8MW.`ڒOypE ^+>ɯ=%绞~ےfZZNg9 Jecυ3ncdOCmqb#t_quٗY@>ש+' R2s%ZhҀ %)> Q6y}A-H:q|Ѻ6룟|_ľ& ݜl!nY1YvTP15!5%D)^e_ p]g? nC.2,yK0CZ%-d|g',f| ſO~>`=#yb7k}o wB|2ҳ5KvwyG? }/K|3hB2Ĺ$>ihR8I琝tepYuewyx;/(u(`]׎\gv,"j.L]ߐ2K  \")EJ =M~w_5]q[+T Qu|/Z^$^L3sf_-y62*f>D3 ܀eCzjF5-2Z?®g d\4lW6)f֯i`y,t`Xo%8ưru} XCRr.ƛ,F< -ize70JN2ΡJ3m.Xos| p )?ε0Dƒ_3m\E n޾/|g߭08b6Rt}ɠGU/~| b&/DWXI`1.YkřfZ^ 1}o%s&R"7/l C5 +,,EJ gڌl1d@e|EПHbb)Qg5OFlU iLJ6X-*q׎_|~Q*J.v+gA$~{BK\ᯛ'Ȣ -*$$TG{C^dz]8z:Γ; tg>v[FQAM3 U!򬒕;lY:0gąWR ?ukMBbcMM>i͟Yv/}. B%)Y[ߋ\j!׀ 7XQh4T+ e$̓F8a`!#Kԃ-Cꦣښ-.=s$&Aɻ4;tط m$ 7Ϊ]]V7"WE)RDž纑K~k{d buƟ|>=4Ӗ!m,| S`tBK"L$5q!;d[ۜ:DTXpқ_4D]MW%ntw7\i. :Dڭ4#:75'qܚ8IwJXbHWHƬx0&RA& zI!Ed*M%e Z'{unѶB y4C%^p \D`R%zc( .~vG%{D4uEׂ|(oH,Wqy2P[2_ q4EQPKGLdf-ZՆq:1r*҆Hѷ-1F5A`FP,O@0>]\&J%KbLlϝq ;b# qR2i:= q^^.^~ k1&o-Q=N@oX9ѵ.qlcVV7J.Zx0&gJW2d~.s6sb:|dwRUr&add2$w ŧbpz3:jE.8qpKc#XIsUr F_2ZN!tj Nfe(kCUotwEk C7w+s)fF,[$84P<+ݦ!q9Jrfdyil>ztNŀDͲP* h(RQYBEVVϭq᫁dO1|)+ !qRjFY X^E (JjZ;"dZ!ulD-0dS2)U-:"F](XJ1MMj+YpQl# ,"+xj.7 Tw/}b>F(v<ąM* fx D؀tt6 |XK4EI iv_F\X6{Ha?*/ms׺ͧt綴 =ty.H-lͰ`Dy gtdO5l̀{O5^#0h'JH2{/ȿjv&$ IxgF؉}/y; *G-={}e I^y !cw&C7drDlNO LW^6i nƐPq~2G JpSh9/Ę U&7nuGk &=>σZ h uFmk&GO u#A& Yf[Bϟ5Lob zWH_l0Po 7 '*}G`G` °2$H@W߻ز>Ucƌ0EfI˔л,);SaE2khM ne~ 1*(lz{\}rڱ#Njq@r E)^!N}CPcVqy̘nh;1 v29Mp8ZxP!VgdbH Ԍz"V@eI"> .*K]K=4BMǟ/ ".)r Ӕ>p#Nac"QVc)L>jVtm.ޏsl(h1 ; lJJ _ݫFu#F+paY.xV/=I?,&8ƣZ8АHTt0|`Kc]@PqpU )`A!YQ8ID-`nK/v9k F}$^"`Gr_:CPZ4lF3ª5QɮGq(Ac?#^k K5p b<6jn8^")j'r;Ddʇu+8H!`AЕrF|X5k) `: 9q56h0ԝH =H->4[)%Q] LUITFCƚEp1)Pk8?A/SDu6mp^\jhESEXOQuyO:˼aX^#hoޔ#\\KQ3J`FX%*Q_{C 8ʌ2:BN4?;%J\jYPPitiGm;c&߼>`-ֆ[)vU* oc 1>3`_y,aX%DXlzepIU;zk`} \Rj̄+U͸;xʥ)aGvb-%u-sb.Xo n\WӇ>_U~;J%}HX6؜uvV%AG+\ Ĵr/x#~H Rm-)lIHJZ'K7"s4MN6c3F:# ^FS3ĶQzu^p7/$a `-ݰ"=$>V< *̄fTO}e 5Ƙ`׊;l,-45T5]F :t)&g.F̜]nnX,T(m pq.<M-V ,j+_:‹ wRXl.8͹~#yEG}Z3"XS4ϵeN0 3`a+r^7IN3l ;b=j@Okg7Lډ*UX*lȬPRρqɺOs"m>&M6+2(U!UaTA) iH^LB-"X7{FJzK6-[Puyf-4#_',Yp-T\xG)`` ߐ 2Ω5>S8=RE ,HE*T [%ȇOEOEfUYtnaU*vأJȒޖ,>!o6(+=Vq)S3*at*a՜F@D:xfPjU{`- i狦j{tQqL&}&(V3k+0TXd Ǭ\`x=KO#ޛV>>iùiZ#@57J&]|fš]Ű8M*P U(ō Q`J7T,\S q"Q^w6UR `2Mv!ؔdEDcΰg+TyWM |Vdhؓni$ܠgULVe-1P xBX;чA57aqdȴ;=0mxXx"xxVQ栢9 ɀWqSUL%!Y2!(N|h&! M!y 9_%dt'xA|0N @1,;6 ۹H"q޵a1)oSy Tmmbfixwvn|'.FQj eƋ&Y(,`,x} wmxq㈱,T+"<`0 C)ߗxH* RΈdYdJF<#>h!s]pH 9Doh`j%?wdף 9@{0&{tB~QEҞqRPKTu1~Nܷpb/p6_,&uu \!÷?q9AENU1 /,U5fFS1pg|ײ[4t䁺J(:Wݫ )ʂ/Xx(TCA(c5 fZG &CFUQqAYX}\?v7tpޏLrN =tu 2 X7r՝.7ocϑY/.x]>e1oݲܠKsWKLŕXњ o~}ۤ&|m/1+ZƬose:^۬26w?M\k0RZ/XlWfCV?ה"6(,f@(Xun'w1.xo/㬝| 75-'qEkp@Z*J#sGU^IMiz%nFGG@YiFuQ&[% VzOsPI}cUaSڙ|̠)˟GJ9 ^ܕKY1OlV` D v*cnly7:M Hݬ! '֯&kf0%92*QԈq.ܲ& 1S<(? #*%MNFk<s4ۉdz\$G;|v6fӊNjTwv ^jܲwujr(c8JLuֿOnhMM%Z,mfs@'ua2r>ylv; 7TřMT aveu"LOB4MQ)=9=yAWugdLjtnlg֣Qo &x7ިS Mg~sQzz!)*"0, LB^w/&dI|N`ՓW$lK8F_/ђ(%y憬p4b@0TU%S>+p SLM3EBbO[xoa`<XTӵĈ&pra=HhMÅec}K{L@ݕ:.b"@'|AaQkmկ^ s<-;EŰW|9L+[\d'.1*PNFȰuݩ^#OI?K~&p{YćirVdcJ&uS^;+|``MPZZ))\}tL:LAjZ 8|6gm4kTB]{IpV>t啧ೃdȔw $*e۸6eA?wf<غE܌TXjI\ז0YrN|A@,XOtCW1 D}VQgIUJ3E'Oxi1}R1ՍyA|A 3K "aa^޸I!X3(E3@z9Yi11qv|;YTt765]TIw wth||9QpUlzotMY|G?`ӫ?4KjykVSm14^ Lѧ<Ã~B>=):`Xtę_<zZy9~>!䩍 j2X~l]d0q*:x&]PlNL /9I9 =є%E'TAWi,Ѝ7BU2qSz*[p 0HmSx[+D~'YjM@ZdV8`Zk#8vʔґ]Iʋnn;+w ~\XeϨHKY>[IR[n*H!2G_}AG >LAXqQ=EwѸH E{zO FN Y51V{-nLR :/XQnv&WkH)զ$~=t{O?~43$fR'#Nj=tU*r<_1A؀ʏΘx7u0"L9EDXpB :y *ؙAZ k:q{ȯ>%8WGаR*8K/>2ϕE5V t G6l/&땡F^`kp`ݰ\Ss=mUeyh|ʫ` D|z`:/b,#ꇊ_M)[M7Ju22[5&:7ő;ɘOG6UPgPR-,NXn>PG5m'ՏHn uN01~J|wL[~x.=q]߁X^Y|e֜_,[t5F޽'Bb^~:vd# 4j9]iH!Յŵ!jnom6ymFN~s=q[B݈S8Z.yvрrO\ǀKQ|iҥk{˅ /$x*G2 J*Y_Z3 [jCNy !$T/KpPum<\~`[4cÆ76CyhFjNX'_rhgh:қzoaT{|:G;Nn E.w8Q8֚1GGMgXel*B֔;9aEj9I=73vWr8*wQDq@A_؝Y3;Bbm iVuTt?Bҵʼ*>XVf1wVGVm $f7orZ2eub8G̵}$|2îUWܚKɃ]U%pc Ҙ =@.gitOf{N gݓ/\W?Wiښ4n8'Zhg#ĂZ|8l;qBeDĸVZ*jAbu{M/൲47[j+29qԶ A}lMZd>>Z=S|ł+ѹY\Z[M'k,5"1Ć!L~+y"Cv(0[F "X x +I)ѝ,:)TmMt-|1*i9=ßX'b@Y{l̞Ŗh:hǗ [e<#˛ d|h9N?ԋRKFG^0X2. MQnfWl1_^?/i@*Qۥ2 hɚxw-kMZtŅQVt3[eB |Y+r{i%z@/?w`=J#Of Q^&{HSչvϰF5ƿfA+x4MC3홢w]EW3`Y>5=]$\dȯf4UH9-cmS9pa`^ɚ~ddxaUSk\ļRT(4NS`--sg*1m.R!U|8n k7tSTL5Mi5ݽ8V<.҈x!=',o4u 4 po`)~<h=?δoÄi-Fh02H ?, ]*r2* .^#-x2G tv^oQ7ݐ]id6lTQ[Hl P3b`c=6+]viZK|95RYF ^*l>HZ ժn˧y",6n`ݏk|\S ]!2c1yrNSzSya;6Te24,YpLvo , zEj%3Y^+sjBt024HWN9SNj^5噫)ʡҫ~(~<0ZQL5C3cA&ҘgҘ ֱ rCf]NŃMxSbsaxĉ='N,nnΘ`$ zCZϲUH9ܠWjj]ѧ+aP`MI԰|8T=I-EyP)ƗY8p֐9Sp6xT)lF՛pw1@G9z·ϰ uӰ!XXeGpJ7AjJPV1T|r Y@Y2[ NXqXyn` 0q)L.۱MO.ԣbO{yX<9 ΚlZa9\qFLC ^X3hm%du5 7pٔ}xq A GCEŃ&///"]z]f ]1[mfEwB>m=ҊA Y->91KWP ],3=PC_}r䓷%D# (6UӿES|状^8XBHl,)]kI9e5)ᅞQ׽D7`| 0>k< V *"9Mյn*aX'>|`E)Tߴ^5[9bVl-ϾKva43W>vbi.ы^P LC2TQqy+N#C Jk&R4uҼ`-pt-Lڭ`¥5H yX%,'viW -*zlPHSt{NN{lh792iaJ9c se`Ӎ|t&~DBSm#^N+)᠔)kڏ'߉3<=8oNGXҀ(DKQ1M׊0`hUҌisTHĎΐnZ kF:ӨzS ֌2` >@/U1ϯz"tǏ?Y O(ʫTS#X,=5 B~=_ DkEuf! X{YuQ:L2Ηp:.xwC]~ ]O{ʋci=BtDC?uUg:J'bZtRgUtF4q1E.]%%cp{id5:┴^/E>1kGq}xtQV*O·}O3kq {3쉤Es:9Rq{XMEzȂVVu[ϴR0Y֨z]K-KJ;vciu4TOۅSCF4GHӃ#B\Z X#$MpҸBV[r<EQQ2t`n_9oc]*uU0UO[ϗʠbIx5,c T-J,Hd yޤ S4*j{ u||xW؎HZ6麬ӣߤI^9'& 3[x4SͲL0lirQ%d)i$Z]k  qeVR02$9)OAi`(ó^KnFFǢv{> klUq>Y"E##Br(J)e`2<k  XX4;Z9ci?wNZ:2jve펋hxVBY􉣵t>HCˋҨԺYKBʾᶎiVyn&s^M!6#d^N1eϰ\\$};|6jB9xy!RlY*"4e˰$Ψao"l򖺞$OO) yRF& "*RnO[OԩQ $wc tEXyEĢ!{HNwQ)V4V"IithQkC 2K1aRV-,YçX$oދޗUnVXc]!4Q0mqf1:EG\LWk!<acong,B]]nߦUcrLCxU%kfm0d&xIi ?/Řg27RBR-U#>YB&yĈ~QR~07@Fѡ-hʣMAL, fV-]kJ XJl9IQ]VĊ>1`K% J˳l)! [Y|o즉ą 0u h;` |f)pV. Ϟ .xMU?hk_ҭЛǔ(8Wc KӰ>,§2)]iK#?--Κ\(XY~c0-ՙJTl:?hO3v=]<@"hp5ӧx'=}MC'?]|P}ѡ4Fd6N[̪&3c? خ9Ez;V(XGPRI7O? ui JlRyiW2t]9>? mʩ6ֽ^M[+λlZcOeʺx? n[GO[vz̥ɇ[ɚ9) zU\~\tQQ`~K? MɴU)EeN+Yo܂'C&- k359"Hcd|)WFkhb)Z6Y ◮^1=-顅'~:>\-Rޘ-zN<N )`4kl`Z[yr"! qG)USɤͬO>)zCy駟zmN-R~Ҋ)8y*J G _WxFz T$OQ1c^LNilXQxkx} {nN,3< nPwaS92MnڦwrgB^I:RǘG@L 9鮔:!F>4x&uq] 9袐F-#sd-,%T}Lbѩ* JχS$FLv3~YS0eM|3 z[^>Z@;y-\ i1oF)W R`?Lk4׭e򳅂Es:?HZ+`/.U5i Dfڴh+dqCyggQuJ=a(ۋ)O%{S)u$gOB{QSj/VhsLWupngcu\*+ԴeNDނ$W#*/'$PdL(ꪓEl޶F<$gxґ):!SZPUt4- M H`hMFCN'w(XLG/ +,: Y҄0'7WrVq:o={WB+IjOj?uu-55ՕUO,"Oc!Xi:jx8Ρ35>,T^[RWR IS%%iI0/s=O?CN_뛴Oyfʔ0~C缫*p=_1:̿]86Esgb>֪j)+]%ط7q*la] Ŗ3g0rb>˅+]TAx-Vb}Xi8CWu1ޟ,k H^荙 GׅB COyD5UZ!$"SZs܆}9zy#MԀ(5]0 5] '~< XƁ9<"Fa86r ϶*6M_Ŕ~%._xZVLJ:QؼpA^լ۱ TafWHJxk -:uذ2{ =rd1#wA+q==1je0e a`LUjja;LĠ/HD:z[m>LԆzAY p?K8~W|E&\X@{juVǭ,M5r"pXeزMy} y`ӎy?XSո߬۲ 4%Ml:C-,s\Q^ڵlCܹu[g3N/5pR/1o2=3"]V8{QXZۤM=n7qw*>x,䒘CdZnZt:[ ,ƥkݠFzCq}@ۂz]qeYBCr: كV|h `[czvǹ=mn*h+[H<-׷ `æ!l-$l^gZg,jհUÆa-zX侮@4%8x ?\78:L|bv?\ s%*j10P-T@aT`Ř滀,~ctV;כVوEmhtB}"*&/නjFUCY5 : -<9ьhw yY4L?VLR^C VKg9!5gQ*z#yŸg$ɢ FrٜXXF{KUNAּgM*%Z~.˦tilBl6|p*"*#VHWe3v) #zN ;;d3:Htm# Мe[m`Q߿ʎ"Kgo t1xkC-JZpU?R7뫪>.xH@+ kEECX`n l[XC5L R6Q6 z^Dy, %<.V.S`oKc%mM}'b=}!G|_χv+aO Hc6*,uŪyEEAr32 URtZv*P aVŇ+ m+xq58՜cz G=IJ0sq "}ۑ=8aGKe-0*=_{2Z/X_z6acdd2 Oc,"áY3nC >Х ʰY4X' fo#}BA<9lX듣.e+GQ5[AKϳI0/cyj?lV%@Zl{ϱ(}x}**@0 &" ۄET[!lK(IBZh a?ϻtW\3 ~n< =W<"sar3FEƉ'ʡ/3&d$:]_Z4sKqraLKi;G1f/]̵h,8 G.ۑ7.MWs7p6,g =@iYgnJ6bps8yiSݴ|yV"/_^?w9SL4i# 8c`5}r]ꙃ-e 2a VUO q6J >̥/L 7f7 *J.|Vy]6G\fsJ7 dlUvd{;}!G4tJ鶹s :qP!1#Y?ýx^DqD6~>ʢ`azgB̲9jqfi.ÇOܵk⮉fl5۵}>fg_%c}n|ta"h<(1Kqf"QGv>Au|ar,%kgьv2PԖ<&LN.' 6/8A.68v2--4uН0l[}GaB;KVE⯆ӴK/EElm^+ܗtPiZFX/3x&s귩Jcح]-xO4m+/2;ˁG/E*g8ꈁl/EqĨIs6mov-OMN!;w/)^摓&M*2^fz TU߾sӁ)FxEf h( IENDB`toro-1.0.1/doc/changelog.rst000066400000000000000000000132301265215761400157320ustar00rootroot00000000000000Changelog ========= .. module:: toro Changes in Version 1.0.1 ------------------------ Bug fix in :class:`~toro.RWLock`: when max_readers > 1 :meth:`~toro.RWLock.release_read` must release one reader in case :meth:`~toro.RWLock.acquire_read` was called at least once:: @gen.coroutine def coro(): lock = toro.RWLock(max_readers=10) assert not lock.locked() yield lock.acquire_read() lock.release_read() But, in old version :meth:`~toro.RWLock.release_read` raises RuntimeException if a lock in unlocked state, even if :meth:`~toro.RWLock.acquire_read` was already called several times. Patch by `Alexander Gridnev `_. Changes in Version 1.0 ---------------------- This is the final release of Toro. Its features are merged into Tornado 4.2. Further development of locks and queues for Tornado coroutines will continue in Tornado. For more information on the end of Toro, `read my article `_. The Tornado changelog has comprehensive instructions on `porting from Toro's locks and queues to Tornado 4.2 locks and queues `_. Toro 1.0 has one new feature, an :class:`.RWLock` contributed by `Alexander Gridnev `_. :class:`.RWLock` has *not* been merged into Tornado. Changes in Version 0.8 ---------------------- Don't depend on "nose" for tests. Improve test quality and coverage. Delete unused method in internal ``_TimeoutFuture`` class. Changes in Version 0.7 ---------------------- Bug fix in :class:`~toro.Semaphore`: after a call to :meth:`~toro.Semaphore.acquire`, :meth:`~toro.Semaphore.wait` should block until another coroutine calls :meth:`~toro.Semaphore.release`:: @gen.coroutine def coro(): sem = toro.Semaphore(1) assert not sem.locked() # A semaphore with initial value of 1 can be acquired once, # then it's locked. sem.acquire() assert sem.locked() # Wait for another coroutine to release the semaphore. yield sem.wait() However, there was a bug and :meth:`~toro.Semaphore.wait` returned immediately if the semaphore had **ever** been unlocked. I'm grateful to `"abing" `_ on GitHub for noticing the bug and contributing a fix. Changes in Version 0.6 ---------------------- :class:`~toro.Queue` now supports floating-point numbers for ``maxsize``. A ``maxsize`` of 1.3 is now equivalent to a ``maxsize`` of 2. Before, it had been treated as infinite. This feature is not intended to be useful, but to maintain an API similar to ``asyncio`` and the standard library Queue. Changes in Version 0.5 ---------------------- Rewritten for Tornado 3. Dropped support for Tornado 2 and Python 2.5. Added support for Tornado 3's Futures_: - All Toro methods that took callbacks no longer take callbacks but return Futures. - All Toro methods that took *optional* callbacks have been split into two methods: one that returns a Future, and a "nowait" method that returns immediately or raises an exception. - :meth:`AsyncResult.get_nowait` can raise :exc:`NotReady` - :meth:`Queue.get_nowait` can raise :exc:`Empty` - :meth:`Queue.put_nowait` can raise :exc:`Full` - All Toro methods that return Futures accept an optional ``deadline`` parameter. Whereas before each Toro class had different behavior after a timeout, all now return a Future that raises :exc:`toro.Timeout` after the deadline. Toro's API aims to be very similar to Tulip_, since Tulip will evolve into the Python 3.4 standard library: - Toro's API has been updated to closely match the locks and queues in Tulip. - The requirement has been dropped that a coroutine that calls :meth:`~toro.Queue.put` resumes only *after* any coroutine it awakens. Similar for :meth:`~toro.Queue.get`. The order in which the two coroutines resume is now unspecified. - A Queue with maxsize 0 (the default) is no longer a "channel" as in Gevent but is an unbounded Queue as in Tulip and the standard library. ``None`` is no longer a valid maxsize. - The ``initial`` argument to Queue() was removed. - maxsize can no longer be changed after a Queue is created. The chief differences between Toro and Tulip are that Toro uses ``yield`` instead of ``yield from``, and that Toro uses absolute deadlines instead of relative timeouts. Additionally, Toro's :class:`~toro.Lock` and :class:`~toro.Semaphore` aren't context managers (they can't be used with a ``with`` statement); instead, the Futures returned from :meth:`~toro.Lock.acquire` and :meth:`~toro.Semaphore.acquire` are context managers. .. _Futures: http://www.tornadoweb.org/en/stable/concurrent.html#tornado.concurrent.Future .. _Tulip: http://code.google.com/p/tulip/ Changes in Version 0.4 ---------------------- Bugfix in :class:`~toro.JoinableQueue`, `JoinableQueue doesn't accept an explicit IOLoop `_. Changes in Version 0.3 ---------------------- Increasing the :attr:`~toro.Queue.maxsize` of a :class:`~toro.Queue` unblocks callbacks waiting on :meth:`~toro.Queue.put`. Travis integration. Changes in Version 0.2 ---------------------- Python 3 support. Bugfix in :class:`~toro.Semaphore`: :meth:`release` shouldn't wake callbacks registered with :meth:`wait` unless no one is waiting for :meth:`acquire`. Fixed error in the "Wait-Notify" table. Added :doc:`examples/lock_example` to docs. Changes in Version 0.1.1 ------------------------ Fixed the docs to render correctly in PyPI. Version 0.1 ----------- First release. toro-1.0.1/doc/classes.rst000066400000000000000000000037321265215761400154460ustar00rootroot00000000000000:mod:`toro` Classes =================== .. currentmodule:: toro .. contents:: Contents :local: .. _primitives: Primitives ~~~~~~~~~~ AsyncResult ----------- .. autoclass:: AsyncResult :members: Lock ---- .. autoclass:: Lock :members: RWLock ------ .. autoclass:: RWLock :members: Semaphore --------- .. autoclass:: Semaphore :members: BoundedSemaphore ---------------- .. autoclass:: BoundedSemaphore :members: Condition --------- .. autoclass:: Condition :members: Event ----- .. autoclass:: Event :members: Queues ~~~~~~ Queue ----- .. autoclass:: Queue :members: PriorityQueue ------------- .. autoclass:: PriorityQueue :members: LifoQueue --------- .. autoclass:: LifoQueue :members: JoinableQueue ------------- .. autoclass:: JoinableQueue :members: Exceptions ~~~~~~~~~~ .. autoclass:: Timeout .. autoclass:: NotReady .. autoclass:: AlreadySet Toro also uses exceptions Empty_ and Full_ from the standard module Queue_. .. _Empty: http://docs.python.org/library/queue.html#Queue.Empty .. _Full: http://docs.python.org/library/queue.html#Queue.Full .. _Queue: http://docs.python.org/library/queue.html Class relationships ~~~~~~~~~~~~~~~~~~~ Toro uses some of its primitives in the implementation of others. For example, :class:`JoinableQueue` is a subclass of :class:`Queue`, and it contains an :class:`Event`. (:class:`AsyncResult` stands alone.) .. graphviz:: digraph Toro { graph [splines=false]; node [shape=record]; // First show UML-style subclass relationships. edge [label=subclass arrowtail=empty arrowhead=none dir=both]; Queue -> PriorityQueue Queue -> LifoQueue Queue -> JoinableQueue Semaphore -> BoundedSemaphore // Now UML-style composition or has-a relationships. edge [label="has a" arrowhead=odiamond arrowtail=none]; Event -> JoinableQueue Condition -> Event Event -> Semaphore Queue -> Semaphore Semaphore -> Lock } toro-1.0.1/doc/conf.py000066400000000000000000000176021265215761400145570ustar00rootroot00000000000000# -*- coding: utf-8 -*- # # Toro documentation build configuration file, created by # sphinx-quickstart on Wed Oct 17 19:46:26 2012. # # This file is execfile()d with the current directory set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. # Custom: Sphinx doesn't produce a useful "module index" for Toro html_use_modindex = False doctest_global_setup = """ import toro """ html_theme_options = {'collapsiblesidebar': True} import sys, os # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path[0:0] = [os.path.abspath('..')] import toro # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode', 'sphinx.ext.graphviz'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'Toro' copyright = u'2012, A. Jesse Jiryu Davis' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = toro.version # The full version, including alpha/beta/rc tags. release = toro.version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = ['_build'] # The reST default role (used for this markup: `text`) to use for all documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'Torodoc' # -- Options for LaTeX output -------------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', # Additional stuff for the LaTeX preamble. #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ('index', 'Toro.tex', u'Toro Documentation', u'A. Jesse Jiryu Davis', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True # -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'toro', u'Toro Documentation', [u'A. Jesse Jiryu Davis'], 1) ] # If true, show URL addresses after external links. #man_show_urls = False # -- Options for Texinfo output ------------------------------------------------ # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ('index', 'Toro', u'Toro Documentation', u'A. Jesse Jiryu Davis', 'Toro', 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. #texinfo_appendices = [] # If false, no module index is generated. #texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' toro-1.0.1/doc/examples/000077500000000000000000000000001265215761400150705ustar00rootroot00000000000000toro-1.0.1/doc/examples/event_example.rst000066400000000000000000000003501265215761400204540ustar00rootroot00000000000000:class:`~toro.Event` example - a caching proxy server ===================================================== .. automodule:: examples.event_example .. literalinclude:: ../../examples/event_example.py :start-after: # start-file toro-1.0.1/doc/examples/index.rst000066400000000000000000000001701265215761400167270ustar00rootroot00000000000000Examples ======== .. toctree:: producer_consumer_example lock_example event_example web_spider_example toro-1.0.1/doc/examples/lock_example.rst000066400000000000000000000003321265215761400202630ustar00rootroot00000000000000:class:`~toro.Lock` example - graceful shutdown =============================================== .. automodule:: examples.lock_example .. literalinclude:: ../../examples/lock_example.py :start-after: # start-file toro-1.0.1/doc/examples/producer_consumer_example.rst000066400000000000000000000003101265215761400230650ustar00rootroot00000000000000Producer-consumer example ========================= .. automodule:: examples.producer_consumer_example .. literalinclude:: ../../examples/producer_consumer_example.py :start-after: # start-file toro-1.0.1/doc/examples/web_spider_example.rst000066400000000000000000000004521265215761400214610ustar00rootroot00000000000000:class:`~toro.Queue` and :class:`~toro.Semaphore` example - a parallel web spider ================================================================================= .. automodule:: examples.web_spider_example .. literalinclude:: ../../examples/web_spider_example.py :start-after: # start-file toro-1.0.1/doc/faq.rst000066400000000000000000000061331265215761400145560ustar00rootroot00000000000000Frequently Asked Questions ========================== .. module:: toro What's it for? -------------- Toro makes it easy for Tornado coroutines--that is, functions decorated with `gen.coroutine`_--to coordinate using Events, Conditions, Queues, and Semaphores. Toro supports patterns in which coroutines wait for notifications from others. .. _gen.coroutine: http://www.tornadoweb.org/en/stable/gen.html#tornado.gen.coroutine Why the name? ------------- A coroutine is often called a "coro", and a library of primitives useful for managing coroutines is called "`coros`_" in Gevent and "`coro`_" in Shrapnel. So I call a library to manage Tornado coroutines "toro". .. _coros: http://www.gevent.org/gevent.coros.html .. _coro: https://github.com/ironport/shrapnel Why do I need synchronization primitives for a single-threaded app? ------------------------------------------------------------------- Protecting an object shared across coroutines is mostly unnecessary in a single-threading Tornado program. For example, a multithreaded app would protect ``counter`` with a `Lock`_:: import threading lock = threading.Lock() counter = 0 def inc(): lock.acquire() counter += 1 lock.release() .. _Lock: http://docs.python.org/library/threading.html#lock-objects This isn't needed in a Tornado coroutine, because the coroutine won't be interrupted until it explicitly yields. Thus Toro is *not* designed to protect shared state. Instead, Toro supports complex coordination among coroutines with :ref:`the-wait-notify-pattern`: Some coroutines wait at particular points in their code for other coroutines to awaken them. Why no RLock? ------------- The standard-library RLock_ (reentrant lock) can be acquired multiple times by a single thread without blocking, reducing the chance of deadlock, especially in recursive functions. The thread currently holding the RLock is the "owning thread." In Toro, simulating a concept like an "owning chain of coroutines" would be over-complicated and under-useful, so there is no RLock, only a :class:`Lock`. .. _RLock: http://docs.python.org/library/threading.html#rlock-objects Has Toro anything to do with Tulip? ----------------------------------- Toro predates Tulip_, which has very similar ideas about coordinating async coroutines using locks and queues. Toro's author implemented Tulip's queues, and version 0.5 of Toro strives to match Tulip's API. The chief differences between Toro and Tulip are that Toro uses ``yield`` instead of ``yield from``, and that Toro uses absolute deadlines instead of relative timeouts. Additionally, Toro's :class:`~toro.Lock` and :class:`~toro.Semaphore` aren't context managers (they can't be used with a ``with`` statement); instead, the Futures returned from :meth:`Lock.acquire` and :meth:`Semaphore.acquire` are context managers: >>> from tornado import gen >>> import toro >>> lock = toro.Lock() >>> >>> @gen.coroutine ... def f(): ... with (yield lock.acquire()): ... assert lock.locked() ... ... assert not lock.locked() .. _Tulip: http://code.google.com/p/tulip/ toro-1.0.1/doc/index.rst000066400000000000000000000071731265215761400151230ustar00rootroot00000000000000======================================================= toro: Synchronization primitives for Tornado coroutines ======================================================= .. module:: toro .. image:: _static/toro.png :align: center .. getting the caption italicized with a hyperlink in it requires some RST hackage *Toro logo by* |musho|_ .. _musho: http://whimsyload.com .. |musho| replace:: *Musho Rodney Alan Greenblat* With Tornado's `gen`_ module, you can turn Python generators into full-featured coroutines, but coordination among these coroutines is difficult without mutexes, semaphores, and queues. Toro provides to Tornado coroutines a set of locking primitives and queues analogous to those that Gevent provides to Greenlets, or that the standard library provides to threads. .. important:: Toro is completed and deprecated; its features have been merged into Tornado. Development of locks and queues for Tornado coroutines continues in Tornado itself. .. _gen: http://www.tornadoweb.org/en/stable/gen.html .. _the-wait-notify-pattern: The Wait / Notify Pattern ========================= Toro's :ref:`primitives ` follow a "wait / notify pattern": one coroutine waits to be notified by another. Let's take :class:`Condition` as an example: .. doctest:: >>> import toro >>> from tornado import ioloop, gen >>> loop = ioloop.IOLoop.current() >>> condition = toro.Condition() >>> @gen.coroutine ... def waiter(): ... print "I'll wait right here" ... yield condition.wait() # Yield a Future ... print "I'm done waiting" ... >>> @gen.coroutine ... def notifier(): ... print "About to notify" ... condition.notify() ... print "Done notifying" ... >>> @gen.coroutine ... def runner(): ... # Yield two Futures; wait for waiter() and notifier() to finish ... yield [waiter(), notifier()] ... loop.stop() ... >>> future = runner(); loop.start() I'll wait right here About to notify Done notifying I'm done waiting Wait-methods take an optional ``deadline`` argument, which is either an absolute timestamp:: loop = ioloop.IOLoop.current() # Wait up to 1 second for a notification yield condition.wait(deadline=loop.time() + 1) ...or a ``datetime.timedelta`` for a deadline relative to the current time:: # Wait up to 1 second yield condition.wait(deadline=datetime.timedelta(seconds=1)) If there's no notification before the deadline, the Toro-specific :class:`Timeout` exception is raised. .. _the-get-put-pattern: The Get / Put Pattern ===================== :class:`Queue` and its subclasses support methods :meth:`Queue.get` and :meth:`Queue.put`. These methods are each both a wait-method **and** a notify-method: * :meth:`Queue.get` waits until there is an available item in the queue, and may notify a coroutine waiting to put an item. * :meth:`Queue.put` waits until the queue has a free slot, and may notify a coroutine waiting to get an item. :meth:`Queue.get` and :meth:`Queue.put` accept deadlines and raise :exc:`Timeout` if the deadline passes. See the :doc:`examples/producer_consumer_example`. Additionally, :class:`JoinableQueue` supports the wait-method :meth:`JoinableQueue.join` and the notify-method :meth:`JoinableQueue.task_done`. Contents ======== .. toctree:: examples/index classes faq changelog Source ====== Is on GitHub: https://github.com/ajdavis/toro Bug Reports and Feature Requests ================================ Also on GitHub: https://github.com/ajdavis/toro/issues Indices and tables ================== * :ref:`genindex` * :ref:`search` toro-1.0.1/examples/000077500000000000000000000000001265215761400143235ustar00rootroot00000000000000toro-1.0.1/examples/__init__.py000066400000000000000000000000001265215761400164220ustar00rootroot00000000000000toro-1.0.1/examples/event_example.py000066400000000000000000000036131265215761400175340ustar00rootroot00000000000000""" An oversimplified caching HTTP proxy - start it, and configure your browser to use localhost:8888 as the proxy server. It doesn't do cookies or redirects, nor does it obey cache-control headers. The point is to demonstrate :class:`~toro.Event`. Imagine a client requests a page, and while the proxy is downloading the page from the external site, a second client requests the same page. Since the page is not yet in cache, an inefficient proxy would launch a second external request. This proxy instead places an :class:`~toro.Event` in the cache, and the second client request waits for the event to be set, thus requiring only a single external request. """ # start-file from tornado import httpclient, gen, ioloop, web import toro class CacheEntry(object): def __init__(self): self.event = toro.Event() self.type = self.body = None cache = {} class ProxyHandler(web.RequestHandler): @web.asynchronous @gen.coroutine def get(self): path = self.request.path entry = cache.get(path) if entry: # Block until the event is set, unless it's set already yield entry.event.wait() else: print path cache[path] = entry = CacheEntry() # Actually fetch the page response = yield httpclient.AsyncHTTPClient().fetch(path) entry.type = response.headers.get('Content-Type', 'text/html') entry.body = response.body entry.event.set() self.set_header('Content-Type', entry.type) self.write(entry.body) self.finish() if __name__ == '__main__': print 'Listening on port 8888' print print 'Configure your web browser to use localhost:8888 as an HTTP Proxy.' print 'Try visiting some web pages and hitting "refresh".' web.Application([('.*', ProxyHandler)], debug=True).listen(8888) ioloop.IOLoop.instance().start() toro-1.0.1/examples/lock_example.py000066400000000000000000000027601265215761400173450ustar00rootroot00000000000000"""Graceful shutdown, an example use case for :class:`~toro.Lock`. ``poll`` continuously fetches http://tornadoweb.org, and after 5 seconds, ``shutdown`` stops the IOLoop. We want any request that ``poll`` has begun to complete before the loop stops, so ``poll`` acquires the lock before starting each HTTP request and releases it when the request completes. ``shutdown`` also acquires the lock before stopping the IOLoop. (Inspired by a post_ to the Tornado mailing list.) .. _post: https://groups.google.com/d/topic/python-tornado/CXg5WwufOvU/discussion """ # start-file import datetime from tornado import ioloop, gen, httpclient import toro lock = toro.Lock() loop = ioloop.IOLoop.current() @gen.coroutine def poll(): client = httpclient.AsyncHTTPClient() while True: with (yield lock.acquire()): print 'Starting request' response = yield client.fetch('http://www.tornadoweb.org/') print response.code # Wait a tenth of a second before next request yield gen.Task(loop.add_timeout, datetime.timedelta(seconds=0.1)) @gen.coroutine def shutdown(): # Get the lock: this ensures poll() isn't in a request when we stop the # loop print 'shutdown() is acquiring the lock' yield lock.acquire() loop.stop() print 'Loop stopped.' if __name__ == '__main__': # Start polling poll() # Arrange to shutdown cleanly 5 seconds from now loop.add_timeout(datetime.timedelta(seconds=5), shutdown) loop.start() toro-1.0.1/examples/producer_consumer_example.py000066400000000000000000000013051265215761400221450ustar00rootroot00000000000000"""A classic producer-consumer example for using :class:`~toro.JoinableQueue`. """ # start-file from tornado import ioloop, gen import toro q = toro.JoinableQueue(maxsize=3) @gen.coroutine def producer(): for item in range(10): print 'Sending', item yield q.put(item) @gen.coroutine def consumer(): while True: item = yield q.get() print '\t\t', 'Got', item q.task_done() if __name__ == '__main__': producer() consumer() loop = ioloop.IOLoop.current() def stop(future): loop.stop() future.result() # Raise error if there is one # block until all tasks are done q.join().add_done_callback(stop) loop.start() toro-1.0.1/examples/web_spider_example.py000066400000000000000000000071641265215761400205430ustar00rootroot00000000000000"""A simple web-spider that crawls all the pages in http://tornadoweb.org. ``spider()`` downloads the page at `base_url` and any pages it links to, recursively. It ignores pages that are not beneath `base_url` hierarchically. This function demos two Toro classes: :class:`~toro.JoinableQueue` and :class:`~toro.BoundedSemaphore`. The :class:`~toro.JoinableQueue` is a work queue; it begins containing only `base_url`, and each discovered URL is added to it. We wait for :meth:`~toro.JoinableQueue.join` to complete before exiting. This ensures that the function as a whole ends when all URLs have been downloaded. The :class:`~toro.BoundedSemaphore` regulates concurrency. We block trying to decrement the semaphore before each download, and increment it after each download completes. """ # start-file import HTMLParser import time import urlparse from datetime import timedelta from tornado import httpclient, gen, ioloop import toro @gen.coroutine def spider(base_url, concurrency): q = toro.JoinableQueue() sem = toro.BoundedSemaphore(concurrency) start = time.time() fetching, fetched = set(), set() @gen.coroutine def fetch_url(): current_url = yield q.get() try: if current_url in fetching: return print 'fetching', current_url fetching.add(current_url) urls = yield get_links_from_url(current_url) fetched.add(current_url) for new_url in urls: # Only follow links beneath the base URL if new_url.startswith(base_url): yield q.put(new_url) finally: q.task_done() sem.release() @gen.coroutine def worker(): while True: yield sem.acquire() # Launch a subtask fetch_url() q.put(base_url) # Start worker, then wait for the work queue to be empty. worker() yield q.join(deadline=timedelta(seconds=300)) assert fetching == fetched print 'Done in %d seconds, fetched %s URLs.' % ( time.time() - start, len(fetched)) @gen.coroutine def get_links_from_url(url): """Download the page at `url` and parse it for links. Returned links have had the fragment after `#` removed, and have been made absolute so, e.g. the URL 'gen.html#tornado.gen.coroutine' becomes 'http://www.tornadoweb.org/en/stable/gen.html'. """ try: response = yield httpclient.AsyncHTTPClient().fetch(url) print 'fetched', url urls = [urlparse.urljoin(url, remove_fragment(new_url)) for new_url in get_links(response.body)] except Exception, e: print e, url raise gen.Return([]) raise gen.Return(urls) def remove_fragment(url): scheme, netloc, url, params, query, fragment = urlparse.urlparse(url) return urlparse.urlunparse((scheme, netloc, url, params, query, '')) def get_links(html): class URLSeeker(HTMLParser.HTMLParser): def __init__(self): HTMLParser.HTMLParser.__init__(self) self.urls = [] def handle_starttag(self, tag, attrs): href = dict(attrs).get('href') if href and tag == 'a': self.urls.append(href) url_seeker = URLSeeker() url_seeker.feed(html) return url_seeker.urls if __name__ == '__main__': import logging logging.basicConfig() loop = ioloop.IOLoop.current() def stop(future): loop.stop() future.result() # Raise error if there is one future = spider('http://www.tornadoweb.org/en/stable/', 10) future.add_done_callback(stop) loop.start() toro-1.0.1/ez_setup.py000066400000000000000000000275731265215761400147330ustar00rootroot00000000000000#!python """Bootstrap setuptools installation If you want to use setuptools in your package's setup.py, just include this file in the same directory with it, and add this to the top of your setup.py:: from ez_setup import use_setuptools use_setuptools() If you want to require a specific version of setuptools, set a download mirror, or use an alternate download directory, you can do so by supplying the appropriate options to ``use_setuptools()``. This file can also be run as a script to install or upgrade setuptools. """ import os import shutil import sys import tempfile import tarfile import optparse import subprocess import platform from distutils import log try: from site import USER_SITE except ImportError: USER_SITE = None DEFAULT_VERSION = "1.4.2" DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/" def _python_cmd(*args): args = (sys.executable,) + args return subprocess.call(args) == 0 def _check_call_py24(cmd, *args, **kwargs): res = subprocess.call(cmd, *args, **kwargs) class CalledProcessError(Exception): pass if not res == 0: msg = "Command '%s' return non-zero exit status %d" % (cmd, res) raise CalledProcessError(msg) vars(subprocess).setdefault('check_call', _check_call_py24) def _install(tarball, install_args=()): # extracting the tarball tmpdir = tempfile.mkdtemp() log.warn('Extracting in %s', tmpdir) old_wd = os.getcwd() try: os.chdir(tmpdir) tar = tarfile.open(tarball) _extractall(tar) tar.close() # going in the directory subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) os.chdir(subdir) log.warn('Now working in %s', subdir) # installing log.warn('Installing Setuptools') if not _python_cmd('setup.py', 'install', *install_args): log.warn('Something went wrong during the installation.') log.warn('See the error message above.') # exitcode will be 2 return 2 finally: os.chdir(old_wd) shutil.rmtree(tmpdir) def _build_egg(egg, tarball, to_dir): # extracting the tarball tmpdir = tempfile.mkdtemp() log.warn('Extracting in %s', tmpdir) old_wd = os.getcwd() try: os.chdir(tmpdir) tar = tarfile.open(tarball) _extractall(tar) tar.close() # going in the directory subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) os.chdir(subdir) log.warn('Now working in %s', subdir) # building an egg log.warn('Building a Setuptools egg in %s', to_dir) _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) finally: os.chdir(old_wd) shutil.rmtree(tmpdir) # returning the result log.warn(egg) if not os.path.exists(egg): raise IOError('Could not build the egg.') def _do_download(version, download_base, to_dir, download_delay): egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg' % (version, sys.version_info[0], sys.version_info[1])) if not os.path.exists(egg): tarball = download_setuptools(version, download_base, to_dir, download_delay) _build_egg(egg, tarball, to_dir) sys.path.insert(0, egg) # Remove previously-imported pkg_resources if present (see # https://bitbucket.org/pypa/setuptools/pull-request/7/ for details). if 'pkg_resources' in sys.modules: del sys.modules['pkg_resources'] import setuptools setuptools.bootstrap_install_from = egg def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, download_delay=15): # making sure we use the absolute path to_dir = os.path.abspath(to_dir) was_imported = 'pkg_resources' in sys.modules or \ 'setuptools' in sys.modules try: import pkg_resources except ImportError: return _do_download(version, download_base, to_dir, download_delay) try: pkg_resources.require("setuptools>=" + version) return except pkg_resources.VersionConflict: e = sys.exc_info()[1] if was_imported: sys.stderr.write( "The required version of setuptools (>=%s) is not available,\n" "and can't be installed while this script is running. Please\n" "install a more recent version first, using\n" "'easy_install -U setuptools'." "\n\n(Currently using %r)\n" % (version, e.args[0])) sys.exit(2) else: del pkg_resources, sys.modules['pkg_resources'] # reload ok return _do_download(version, download_base, to_dir, download_delay) except pkg_resources.DistributionNotFound: return _do_download(version, download_base, to_dir, download_delay) def _clean_check(cmd, target): """ Run the command to download target. If the command fails, clean up before re-raising the error. """ try: subprocess.check_call(cmd) except subprocess.CalledProcessError: if os.access(target, os.F_OK): os.unlink(target) raise def download_file_powershell(url, target): """ Download the file at url to target using Powershell (which will validate trust). Raise an exception if the command cannot complete. """ target = os.path.abspath(target) cmd = [ 'powershell', '-Command', "(new-object System.Net.WebClient).DownloadFile(%(url)r, %(target)r)" % vars(), ] _clean_check(cmd, target) def has_powershell(): if platform.system() != 'Windows': return False cmd = ['powershell', '-Command', 'echo test'] devnull = open(os.path.devnull, 'wb') try: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except: return False finally: devnull.close() return True download_file_powershell.viable = has_powershell def download_file_curl(url, target): cmd = ['curl', url, '--silent', '--output', target] _clean_check(cmd, target) def has_curl(): cmd = ['curl', '--version'] devnull = open(os.path.devnull, 'wb') try: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except: return False finally: devnull.close() return True download_file_curl.viable = has_curl def download_file_wget(url, target): cmd = ['wget', url, '--quiet', '--output-document', target] _clean_check(cmd, target) def has_wget(): cmd = ['wget', '--version'] devnull = open(os.path.devnull, 'wb') try: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except: return False finally: devnull.close() return True download_file_wget.viable = has_wget def download_file_insecure(url, target): """ Use Python to download the file, even though it cannot authenticate the connection. """ try: from urllib.request import urlopen except ImportError: from urllib2 import urlopen src = dst = None try: src = urlopen(url) # Read/write all in one block, so we don't create a corrupt file # if the download is interrupted. data = src.read() dst = open(target, "wb") dst.write(data) finally: if src: src.close() if dst: dst.close() download_file_insecure.viable = lambda: True def get_best_downloader(): downloaders = [ download_file_powershell, download_file_curl, download_file_wget, download_file_insecure, ] for dl in downloaders: if dl.viable(): return dl def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, delay=15, downloader_factory=get_best_downloader): """Download setuptools from a specified location and return its filename `version` should be a valid setuptools version number that is available as an egg for download under the `download_base` URL (which should end with a '/'). `to_dir` is the directory where the egg will be downloaded. `delay` is the number of seconds to pause before an actual download attempt. ``downloader_factory`` should be a function taking no arguments and returning a function for downloading a URL to a target. """ # making sure we use the absolute path to_dir = os.path.abspath(to_dir) tgz_name = "setuptools-%s.tar.gz" % version url = download_base + tgz_name saveto = os.path.join(to_dir, tgz_name) if not os.path.exists(saveto): # Avoid repeated downloads log.warn("Downloading %s", url) downloader = downloader_factory() downloader(url, saveto) return os.path.realpath(saveto) def _extractall(self, path=".", members=None): """Extract all members from the archive to the current working directory and set owner, modification time and permissions on directories afterwards. `path' specifies a different directory to extract to. `members' is optional and must be a subset of the list returned by getmembers(). """ import copy import operator from tarfile import ExtractError directories = [] if members is None: members = self for tarinfo in members: if tarinfo.isdir(): # Extract directories with a safe mode. directories.append(tarinfo) tarinfo = copy.copy(tarinfo) tarinfo.mode = 448 # decimal for oct 0700 self.extract(tarinfo, path) # Reverse sort directories. if sys.version_info < (2, 4): def sorter(dir1, dir2): return cmp(dir1.name, dir2.name) directories.sort(sorter) directories.reverse() else: directories.sort(key=operator.attrgetter('name'), reverse=True) # Set correct owner, mtime and filemode on directories. for tarinfo in directories: dirpath = os.path.join(path, tarinfo.name) try: self.chown(tarinfo, dirpath) self.utime(tarinfo, dirpath) self.chmod(tarinfo, dirpath) except ExtractError: e = sys.exc_info()[1] if self.errorlevel > 1: raise else: self._dbg(1, "tarfile: %s" % e) def _build_install_args(options): """ Build the arguments to 'python setup.py install' on the setuptools package """ install_args = [] if options.user_install: if sys.version_info < (2, 6): log.warn("--user requires Python 2.6 or later") raise SystemExit(1) install_args.append('--user') return install_args def _parse_args(): """ Parse the command line for options """ parser = optparse.OptionParser() parser.add_option( '--user', dest='user_install', action='store_true', default=False, help='install in user site package (requires Python 2.6 or later)') parser.add_option( '--download-base', dest='download_base', metavar="URL", default=DEFAULT_URL, help='alternative URL from where to download the setuptools package') parser.add_option( '--insecure', dest='downloader_factory', action='store_const', const=lambda: download_file_insecure, default=get_best_downloader, help='Use internal, non-validating downloader' ) options, args = parser.parse_args() # positional arguments are ignored return options def main(version=DEFAULT_VERSION): """Install or upgrade setuptools and EasyInstall""" options = _parse_args() tarball = download_setuptools(download_base=options.download_base, downloader_factory=options.downloader_factory) return _install(tarball, _build_install_args(options)) if __name__ == '__main__': sys.exit(main()) toro-1.0.1/requirements.txt000066400000000000000000000000151265215761400157650ustar00rootroot00000000000000tornado >= 3 toro-1.0.1/setup.py000066400000000000000000000034171265215761400142240ustar00rootroot00000000000000# Don't force people to install setuptools unless # we have to. try: from setuptools import setup except ImportError: from ez_setup import use_setuptools use_setuptools() from setuptools import setup import sys try: # Work around bug in Python 2.6, TypeError on shutdown. import multiprocessing except ImportError: pass classifiers = """\ Intended Audience :: Developers License :: OSI Approved :: Apache Software License Development Status :: 4 - Beta Natural Language :: English Programming Language :: Python :: 2 Programming Language :: Python :: 2.6 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 Programming Language :: Python :: 3.2 Programming Language :: Python :: 3.3 Programming Language :: Python :: 3.4 Operating System :: MacOS :: MacOS X Operating System :: Unix Programming Language :: Python Programming Language :: Python :: Implementation :: CPython Programming Language :: Python :: Implementation :: PyPy """ description = 'Synchronization primitives for Tornado coroutines.' long_description = open("README.rst").read() major, minor = sys.version_info[:2] kwargs = {} if major >= 3: kwargs['use_2to3'] = True packages = ['toro'] if "test" in sys.argv: packages.append('test') setup(name='toro', version='1.0.1', packages=packages, description=description, long_description=long_description, author='A. Jesse Jiryu Davis', author_email='jesse@emptysquare.net', url='http://github.com/ajdavis/toro/', install_requires=['tornado >= 3'], license='http://www.apache.org/licenses/LICENSE-2.0', classifiers=filter(None, classifiers.split('\n')), keywords='tornado coroutines semaphore mutex queue asynchronous', test_suite='test', **kwargs ) toro-1.0.1/test/000077500000000000000000000000001265215761400134645ustar00rootroot00000000000000toro-1.0.1/test/__init__.py000066400000000000000000000046671265215761400156120ustar00rootroot00000000000000import contextlib from datetime import timedelta from functools import partial from tornado.concurrent import Future from tornado.ioloop import IOLoop from tornado.testing import gen_test, gen import toro @contextlib.contextmanager def assert_raises(exc_class): """Roughly a backport of Python 2.7's TestCase.assertRaises""" try: yield except exc_class: pass else: assert False, "%s not raised" % exc_class def make_callback(key, history): def callback(future): exc = future.exception() if exc: history.append(exc.__class__.__name__) else: history.append(key) return callback def pause(deadline): future = Future() IOLoop.current().add_timeout(deadline, partial(future.set_result, None)) return future class ContextManagerTestsMixin(object): """Test a Toro object's behavior with the "with" statement Combine this mixin with an AsyncTestCase that has a field 'toro_class' """ @gen_test def test_context_manager(self): toro_obj = self.toro_class() with (yield toro_obj.acquire()) as yielded: self.assertTrue(toro_obj.locked()) self.assertTrue(yielded is None) self.assertFalse(toro_obj.locked()) @gen_test def test_context_manager_exception(self): toro_obj = self.toro_class() with assert_raises(ZeroDivisionError): with (yield toro_obj.acquire()): 1 / 0 # Context manager released toro_obj self.assertFalse(toro_obj.locked()) @gen_test def test_context_manager_contended(self): toro_obj = toro.Semaphore() history = [] n_coroutines = 10 @gen.coroutine def f(i): with (yield toro_obj.acquire()): history.append('acquired %d' % i) yield pause(timedelta(seconds=0.01)) history.append('releasing %d' % i) yield [f(i) for i in range(n_coroutines)] expected_history = [] for i in range(n_coroutines): expected_history.extend(['acquired %d' % i, 'releasing %d' % i]) self.assertEqual(expected_history, history) def test_context_manager_misuse(self): toro_obj = self.toro_class() # Ensure we catch a "with toro_obj", which should be # "with (yield toro_obj)" with assert_raises(RuntimeError): with toro_obj: pass toro-1.0.1/test/test_async_result.py000066400000000000000000000055061265215761400176160ustar00rootroot00000000000000""" Test toro.AsyncResult. """ from datetime import timedelta from functools import partial import time from tornado.testing import gen_test, AsyncTestCase import toro from test import make_callback, assert_raises class TestAsyncResult(AsyncTestCase): def test_str(self): result = toro.AsyncResult() str(result) result.set('fizzle') self.assertTrue('fizzle' in str(result)) self.assertFalse('waiters' in str(result)) result = toro.AsyncResult() result.get() self.assertTrue('waiters' in str(result)) def test_get_nowait(self): self.assertRaises(toro.NotReady, toro.AsyncResult().get_nowait) @gen_test def test_raises_after_timeout(self): start = time.time() with assert_raises(toro.Timeout): async_result = toro.AsyncResult(self.io_loop) yield async_result.get(deadline=timedelta(seconds=0.1)) duration = time.time() - start self.assertAlmostEqual(0.1, duration, places=1) @gen_test def test_set(self): result = toro.AsyncResult(io_loop=self.io_loop) self.assertFalse(result.ready()) self.io_loop.add_timeout( time.time() + 0.1, partial(result.set, 'hello')) start = time.time() value = yield result.get() duration = time.time() - start self.assertAlmostEqual(0.1, duration, places=1) self.assertTrue(result.ready()) self.assertEqual('hello', value) # Second and third get()'s work too self.assertEqual('hello', (yield result.get())) self.assertEqual('hello', (yield result.get())) # Non-blocking get() works self.assertEqual('hello', result.get_nowait()) # set() only allowed once self.assertRaises(toro.AlreadySet, result.set, 'whatever') def test_get_callback(self): # Test that callbacks registered with get() run immediately after set() result = toro.AsyncResult(io_loop=self.io_loop) history = [] result.get().add_done_callback(make_callback('get1', history)) result.get().add_done_callback(make_callback('get2', history)) result.set('foo') history.append('set') self.assertEqual(['get1', 'get2', 'set'], history) @gen_test def test_get_timeout(self): result = toro.AsyncResult(io_loop=self.io_loop) start = time.time() with assert_raises(toro.Timeout): yield result.get(deadline=timedelta(seconds=0.1)) duration = time.time() - start self.assertAlmostEqual(0.1, duration, places=1) self.assertFalse(result.ready()) # Timed-out waiter doesn't cause error result.set('foo') self.assertTrue(result.ready()) value = yield result.get(deadline=timedelta(seconds=.01)) self.assertEqual('foo', value) toro-1.0.1/test/test_condition.py000066400000000000000000000101631265215761400170640ustar00rootroot00000000000000""" Test toro.Condition. """ from datetime import timedelta import time from tornado import gen from tornado.testing import gen_test, AsyncTestCase import toro from test import make_callback, assert_raises class TestCondition(AsyncTestCase): def test_str(self): c = toro.Condition() self.assertTrue('Condition' in str(c)) self.assertFalse('waiters' in str(c)) c.wait() self.assertTrue('waiters' in str(c)) @gen_test def test_notify(self): c = toro.Condition(self.io_loop) self.io_loop.add_timeout(time.time() + 0.1, c.notify) yield c.wait() def test_notify_1(self): c = toro.Condition() history = [] c.wait().add_done_callback(make_callback('wait1', history)) c.wait().add_done_callback(make_callback('wait2', history)) c.notify(1) history.append('notify1') c.notify(1) history.append('notify2') self.assertEqual(['wait1', 'notify1', 'wait2', 'notify2'], history) def test_notify_n(self): c = toro.Condition() history = [] for i in range(6): c.wait().add_done_callback(make_callback(i, history)) c.notify(3) # Callbacks execute in the order they were registered self.assertEqual(list(range(3)), history) c.notify(1) self.assertEqual(list(range(4)), history) c.notify(2) self.assertEqual(list(range(6)), history) def test_notify_all(self): c = toro.Condition() history = [] for i in range(4): c.wait().add_done_callback(make_callback(i, history)) c.notify_all() history.append('notify_all') # Callbacks execute in the order they were registered self.assertEqual( list(range(4)) + ['notify_all'], history) @gen_test def test_wait_timeout(self): c = toro.Condition(self.io_loop) st = time.time() with assert_raises(toro.Timeout): yield c.wait(deadline=timedelta(seconds=0.1)) duration = time.time() - st self.assertAlmostEqual(0.1, duration, places=1) @gen_test def test_wait_timeout_preempted(self): c = toro.Condition(self.io_loop) st = time.time() # This fires before the wait times out self.io_loop.add_timeout(st + .1, c.notify) yield c.wait(deadline=timedelta(seconds=0.2)) duration = time.time() - st # Verify we were awakened by c.notify(), not by timeout self.assertAlmostEqual(0.1, duration, places=1) @gen_test def test_notify_n_with_timeout(self): # Register callbacks 0, 1, 2, and 3. Callback 1 has a timeout. # Wait for that timeout to expire, then do notify(2) and make # sure everyone runs. Verifies that a timed-out callback does # not count against the 'n' argument to notify(). c = toro.Condition(self.io_loop) st = time.time() history = [] c.wait().add_done_callback(make_callback(0, history)) c.wait(deadline=timedelta(seconds=.1)).add_done_callback( make_callback(1, history)) c.wait().add_done_callback(make_callback(2, history)) c.wait().add_done_callback(make_callback(3, history)) # Wait for callback 1 to time out yield gen.Task(self.io_loop.add_timeout, st + 0.2) self.assertEqual(['Timeout'], history) c.notify(2) self.assertEqual(['Timeout', 0, 2], history) c.notify() self.assertEqual(['Timeout', 0, 2, 3], history) @gen_test def test_notify_all_with_timeout(self): c = toro.Condition(self.io_loop) st = time.time() history = [] c.wait().add_done_callback(make_callback(0, history)) c.wait(deadline=timedelta(seconds=.1)).add_done_callback( make_callback(1, history)) c.wait().add_done_callback(make_callback(2, history)) # Wait for callback 1 to time out yield gen.Task(self.io_loop.add_timeout, st + 0.2) self.assertEqual(['Timeout'], history) c.notify_all() self.assertEqual(['Timeout', 0, 2], history) toro-1.0.1/test/test_event.py000066400000000000000000000032061265215761400162170ustar00rootroot00000000000000""" Test toro.Event. Adapted from Gevent's lock_tests.py. """ from datetime import timedelta import time from tornado import gen from tornado.testing import gen_test, AsyncTestCase import toro from test import assert_raises class TestEvent(AsyncTestCase): def test_str(self): event = toro.Event() self.assertTrue('clear' in str(event)) self.assertFalse('set' in str(event)) event.set() self.assertFalse('clear' in str(event)) self.assertTrue('set' in str(event)) @gen.coroutine def _test_event(self, n): e = toro.Event() futures = [] for i in range(n): futures.append(e.wait()) e.set() e.clear() yield futures @gen_test def test_event_1(self): yield self._test_event(1) @gen_test def test_event_100(self): yield self._test_event(100) @gen_test def test_event_timeout(self): e = toro.Event() st = time.time() with assert_raises(toro.Timeout): yield e.wait(deadline=timedelta(seconds=0.1)) duration = time.time() - st self.assertAlmostEqual(0.1, duration, places=1) # After a timed-out waiter, normal operation works st = time.time() self.io_loop.add_timeout(st + 0.1, e.set) result = yield e.wait(deadline=timedelta(seconds=1)) duration = time.time() - st self.assertAlmostEqual(0.1, duration, places=1) self.assertEqual(None, result) @gen_test def test_event_nowait(self): e = toro.Event() e.set() self.assertEqual(True, e.is_set()) yield e.wait() toro-1.0.1/test/test_lock.py000066400000000000000000000054441265215761400160340ustar00rootroot00000000000000""" Test toro.Lock. Adapted from Gevent's lock_tests.py. """ from datetime import timedelta import time from tornado import gen from tornado.testing import gen_test, AsyncTestCase import toro from test import make_callback, assert_raises, ContextManagerTestsMixin # Adapted from Gevent's lock_tests.py. class LockTests(AsyncTestCase): def test_acquire_release(self): lock = toro.Lock() self.assertFalse(lock.locked()) self.assertTrue(lock.acquire()) self.assertTrue(lock.locked()) lock.release() self.assertFalse(lock.locked()) @gen_test def test_acquire_contended(self): lock = toro.Lock() self.assertTrue(lock.acquire()) N = 5 @gen.coroutine def f(): yield lock.acquire() lock.release() futures = [f() for _ in range(N)] lock.release() yield futures @gen_test def test_reacquire(self): # Lock needs to be released before re-acquiring. lock = toro.Lock() phase = [] @gen.coroutine def f(): yield lock.acquire() self.assertTrue(lock.locked()) phase.append(None) yield lock.acquire() self.assertTrue(lock.locked()) phase.append(None) future = f() while len(phase) == 0: yield gen.Task(self.io_loop.add_callback) self.assertEqual(len(phase), 1) lock.release() yield future self.assertEqual(len(phase), 2) # Not adapted from Gevent's tests, written just for Toro class LockTests2(AsyncTestCase): def test_str(self): lock = toro.Lock() # No errors in various states str(lock) lock.acquire() str(lock) @gen_test def test_acquire_timeout(self): lock = toro.Lock() self.assertTrue(lock.acquire()) self.assertTrue(lock.locked()) st = time.time() with assert_raises(toro.Timeout): yield lock.acquire(deadline=timedelta(seconds=0.1)) duration = time.time() - st self.assertAlmostEqual(0.1, duration, places=1) self.assertTrue(lock.locked()) @gen_test def test_acquire_callback(self): lock = toro.Lock() history = [] lock.acquire().add_done_callback(make_callback('acquire1', history)) lock.acquire().add_done_callback(make_callback('acquire2', history)) lock.release() history.append('release') self.assertEqual(['acquire1', 'acquire2', 'release'], history) def test_multi_release(self): lock = toro.Lock() lock.acquire() lock.release() self.assertRaises(RuntimeError, lock.release) class LockContextManagerTest(ContextManagerTestsMixin, AsyncTestCase): toro_class = toro.Lock toro-1.0.1/test/test_queue.py000066400000000000000000000334651265215761400162340ustar00rootroot00000000000000""" Test toro.Queue. There are three sections, one each for tests that are 1. adapted from Gevent's test_queue.py, except for FailingQueueTest which isn't applicable 2. adapted from Gevent's test__queue.py, 3. written specifically for Toro. """ import time from datetime import timedelta from Queue import Empty, Full from tornado import gen from tornado.ioloop import IOLoop from tornado.testing import gen_test, AsyncTestCase import toro from test import make_callback, assert_raises, pause # TODO: update from tulip tests # SECTION 1: Tests adapted from Gevent's test_queue.py (single underscore) QUEUE_SIZE = 5 class QueueTest1(AsyncTestCase): type2test = toro.Queue @gen.coroutine def simple_queue_test(self, q): if not q.empty(): raise RuntimeError("Call this function with an empty queue") # I guess we better check things actually queue correctly a little :) q.put_nowait(111) q.put_nowait(333) q.put_nowait(222) target_order = dict(Queue=[111, 333, 222], LifoQueue=[222, 333, 111], PriorityQueue=[111, 222, 333]) actual_order = [q.get_nowait(), q.get_nowait(), q.get_nowait()] self.assertEqual(actual_order, target_order[q.__class__.__name__], "Didn't seem to queue the correct data!") for i in range(QUEUE_SIZE-1): q.put_nowait(i) self.assertTrue(not q.empty(), "Queue should not be empty") self.assertTrue(not q.full(), "Queue should not be full") q.put_nowait(444) self.assertTrue(q.full(), "Queue should be full") try: q.put_nowait(555) self.fail("Didn't appear to block with a full queue") except Full: pass with assert_raises(toro.Timeout): yield q.put(555, deadline=timedelta(seconds=0.01)) self.assertEqual(q.qsize(), QUEUE_SIZE) # Empty it for i in range(QUEUE_SIZE): q.get_nowait() self.assertTrue(q.empty(), "Queue should be empty") try: q.get_nowait() self.fail("Didn't appear to block with an empty queue") except Empty: pass with assert_raises(toro.Timeout): yield q.get(deadline=timedelta(seconds=0.01)) @gen_test def test_simple_queue(self): # Do it a couple of times on the same queue. # Done twice to make sure works with same instance reused. q = self.type2test(QUEUE_SIZE) yield self.simple_queue_test(q) yield self.simple_queue_test(q) class LifoQueueTest1(QueueTest1): type2test = toro.LifoQueue class PriorityQueueTest1(QueueTest1): type2test = toro.PriorityQueue class TestJoinableQueue1(AsyncTestCase): def setUp(self): super(TestJoinableQueue1, self).setUp() self.cum = 0 def test_queue_task_done(self): # Test to make sure a queue task completed successfully. q = toro.JoinableQueue() try: q.task_done() except ValueError: pass else: self.fail("Did not detect task count going negative") @gen.coroutine def worker(self, q): while True: x = yield q.get() if x is None: q.task_done() break self.cum += x q.task_done() @gen.coroutine def queue_join_test(self, q): self.cum = 0 for i in (0,1): self.worker(q) for i in xrange(100): q.put(i) yield q.join() self.assertEqual(self.cum, sum(range(100)), "q.join() did not block until all tasks were done") for i in (0,1): q.put(None) # instruct the tasks to end yield q.join() # verify that you can join twice @gen_test def test_queue_join(self): # Test that a queue join()s successfully, and before anything else # (done twice for insurance). q = toro.JoinableQueue() yield self.queue_join_test(q) yield self.queue_join_test(q) try: q.task_done() except ValueError: pass else: self.fail("Did not detect task count going negative") # SECTION 2: Tests adapted from Gevent's test__queue.py (double underscore) class TestQueue2(AsyncTestCase): def test_repr(self): # No exceptions str(toro.Queue()) repr(toro.Queue()) @gen_test def test_send_first(self): q = toro.Queue() yield q.put('hi') self.assertEqual('hi', (yield q.get())) @gen_test def test_send_last(self): q = toro.Queue() @gen.coroutine def f(): val = yield q.get() self.assertEqual('hi2', val) yield q.put('ok') # Start a task; blocks on get() until we do a put() f() yield q.put('hi2') self.assertEqual('ok', (yield q.get())) @gen_test def test_max_size(self): q = toro.Queue(2) results = [] @gen.coroutine def putter(): yield q.put('a') results.append('a') yield q.put('b') results.append('b') yield q.put('c') results.append('c') future = putter() yield pause(timedelta(seconds=.01)) self.assertEqual(results, ['a', 'b']) self.assertEqual((yield q.get()), 'a') yield pause(timedelta(seconds=.01)) self.assertEqual(results, ['a', 'b', 'c']) self.assertEqual((yield q.get()), 'b') self.assertEqual((yield q.get()), 'c') yield future @gen_test def test_multiple_waiters(self): # tests that multiple waiters get their results back q = toro.Queue() @gen.coroutine def waiter(q, evt): evt.set((yield q.get())) sendings = ['1', '2', '3', '4'] evts = [toro.AsyncResult() for x in sendings] for i, x in enumerate(sendings): waiter(q, evts[i]) # start task @gen.coroutine def collect_pending_results(): results = set() for e in evts: if e.ready(): # Won't block x = yield e.get() results.add(x) raise gen.Return(len(results)) yield q.put(sendings[0]) yield pause(timedelta(seconds=.01)) self.assertEqual((yield collect_pending_results()), 1) yield q.put(sendings[1]) yield pause(timedelta(seconds=.01)) self.assertEqual((yield collect_pending_results()), 2) yield q.put(sendings[2]) yield q.put(sendings[3]) yield pause(timedelta(seconds=.01)) self.assertEqual((yield collect_pending_results()), 4) @gen_test def test_senders_that_die(self): q = toro.Queue() @gen.coroutine def do_send(q): yield q.put('sent') future = do_send(q) self.assertEqual((yield q.get()), 'sent') yield future class TestJoinEmpty2(AsyncTestCase): @gen_test def test_issue_45(self): # Test that join() exits immediately if not jobs were put into the queue # From Gevent's test_issue_45() self.switch_expected = False q = toro.JoinableQueue() yield q.join() # SECTION 3: Tests written specifically for Toro def bad_get_callback(_): raise Exception('Intentional exception in get callback') def bad_put_callback(_): raise Exception('Intentional exception in put callback') class TestQueue3(AsyncTestCase): def test_str(self): self.assertTrue('Queue' in str(toro.Queue())) self.assertTrue('maxsize=11' in str(toro.Queue(11))) q = toro.Queue() for i in range(7): q.get() self.assertTrue('getters[7]' in str(q)) q = toro.Queue(1) for i in range(5): q.put('foo') self.assertTrue('putters[4]' in str(q)) q = toro.Queue(1) self.assertFalse('queue=' in str(q)) q.put('foo') self.assertTrue('queue=' in str(q)) def test_maxsize(self): self.assertRaises(TypeError, toro.Queue, None) self.assertRaises(ValueError, toro.Queue, -1) def test_full(self): q = toro.Queue() self.assertFalse(q.full()) self.assertEqual(q.maxsize, 0) q = toro.Queue(1) self.assertEqual(q.maxsize, 1) self.assertFalse(q.full()) q.put('foo') self.assertTrue(q.full()) def test_callback_checking(self): self.assertRaises(TypeError, toro.Queue().get, callback='foo') self.assertRaises(TypeError, toro.Queue().get, callback=1) def test_io_loop(self): global_loop = self.io_loop custom_loop = IOLoop() self.assertNotEqual(global_loop, custom_loop) q = toro.Queue(io_loop=custom_loop) def callback(future): assert future.result() == 'foo' custom_loop.stop() custom_loop.close(all_fds=True) q.get().add_done_callback(callback) q.put('foo') custom_loop.start() @gen_test def test_float_maxsize(self): # Adapted from asyncio's test_float_maxsize. q = toro.Queue(maxsize=1.3, io_loop=self.io_loop) q.put_nowait(1) q.put_nowait(2) self.assertTrue(q.full()) self.assertRaises(Full, q.put_nowait, 3) q = toro.Queue(maxsize=1.3, io_loop=self.io_loop) yield q.put(1) yield q.put(2) self.assertTrue(q.full()) @gen_test def test_get_nowait_unblocks_putter(self): q = toro.Queue(maxsize=1) q.put_nowait(1) future = q.put(2, deadline=timedelta(seconds=1)) self.assertEqual(1, q.get_nowait()) yield future @gen_test def test_put_nowait_unblocks_getter(self): q = toro.Queue(maxsize=1) future = q.get(deadline=timedelta(seconds=1)) q.put_nowait(1) result = yield future self.assertEqual(1, result) class TestQueueTimeouts3(AsyncTestCase): @gen_test def test_get_timeout(self): q = toro.Queue() st = time.time() with assert_raises(toro.Timeout): yield q.get(deadline=timedelta(seconds=0.1)) duration = time.time() - st self.assertAlmostEqual(0.1, duration, places=1) # Make sure that putting and getting a value returns Queue to initial # state q.put(1) self.assertEqual( 1, (yield q.get(deadline=timedelta(seconds=0.1)))) st = time.time() with assert_raises(toro.Timeout): yield q.get(deadline=timedelta(seconds=0.1)) duration = time.time() - st self.assertAlmostEqual(0.1, duration, places=1) @gen_test def test_put_timeout(self): q = toro.Queue(1) q.put(1) st = time.time() with assert_raises(toro.Timeout): yield q.put(2, deadline=timedelta(seconds=0.1)) duration = time.time() - st self.assertAlmostEqual(0.1, duration, places=1) # Make sure that getting and putting a value returns Queue to initial # state self.assertEqual(1, (yield q.get())) yield q.put(1, deadline=timedelta(seconds=0.1)) st = time.time() with assert_raises(toro.Timeout): yield q.put(2, deadline=timedelta(seconds=0.1)) duration = time.time() - st self.assertAlmostEqual(0.1, duration, places=1) class TestJoinableQueue3(AsyncTestCase): def test_str(self): q = toro.JoinableQueue() self.assertTrue('JoinableQueue' in str(q)) self.assertFalse('tasks' in str(q)) q.put('foo') self.assertTrue('tasks' in str(q)) @gen_test def test_queue_join(self): q = toro.JoinableQueue() yield q.put('foo') yield q.put('bar') self.assertEqual(2, q.unfinished_tasks) future = q.join() q.task_done() self.assertEqual(1, q.unfinished_tasks) q.task_done() self.assertEqual(0, q.unfinished_tasks) yield future @gen_test def test_queue_join_callback(self): # Test that callbacks passed to join() run immediately after task_done() q = toro.JoinableQueue() history = [] q.put('foo') q.put('foo') q.join().add_done_callback(make_callback('join', history)) q.task_done() history.append('task_done1') q.task_done() history.append('task_done2') self.assertEqual(['task_done1', 'join', 'task_done2'], history) @gen_test def test_queue_join_timeout(self): q = toro.JoinableQueue() q.put(1) st = time.time() with assert_raises(toro.Timeout): yield q.join(deadline=timedelta(seconds=0.1)) duration = time.time() - st self.assertAlmostEqual(0.1, duration, places=1) self.assertEqual(1, q.unfinished_tasks) def test_io_loop(self): global_loop = self.io_loop custom_loop = IOLoop() self.assertNotEqual(global_loop, custom_loop) q = toro.JoinableQueue(io_loop=custom_loop) def callback(future): assert future.result() == 'foo' custom_loop.stop() custom_loop.close(all_fds=True) q.get().add_done_callback(callback) q.put('foo') custom_loop.start() @gen_test def test_queue_join_clear(self): # Verify that join() blocks again after a task is added q = toro.JoinableQueue() q.put_nowait('foo') q.task_done() # The _finished Event is set yield q.join() yield q.join() # Unset the event q.put_nowait('bar') with assert_raises(toro.Timeout): yield q.join(deadline=timedelta(seconds=0.1)) q.task_done() yield q.join() # The Event is set again. toro-1.0.1/test/test_rwlock.py000066400000000000000000000252101265215761400163760ustar00rootroot00000000000000""" Test toro.RWLock. """ from functools import partial from random import randint from datetime import timedelta import time from tornado import gen from tornado.testing import gen_test, AsyncTestCase import toro from test import make_callback, assert_raises, ContextManagerTestsMixin # Adapted from toro.test_lock.py class RWLockReadTests(AsyncTestCase): def test_acquire_release(self): lock = toro.RWLock(max_readers=1) self.assertFalse(lock.locked()) self.assertTrue(lock.acquire_read().done()) self.assertTrue(lock.locked()) lock.release_read() self.assertFalse(lock.locked()) def test_release_unlocked(self): lock = toro.RWLock(max_readers=1) with assert_raises(RuntimeError): lock.release_read() @gen_test def test_acquire_contended(self): lock = toro.RWLock(max_readers=1) self.assertTrue(lock.acquire_read().done()) N = 5 @gen.coroutine def f(): yield lock.acquire_read() lock.release_read() futures = [f() for _ in range(N)] lock.release_read() yield futures @gen_test def test_reacquire(self): # Lock needs to be released before re-acquiring. lock = toro.RWLock(max_readers=1) phase = [] @gen.coroutine def f(): yield lock.acquire_read() self.assertTrue(lock.locked()) phase.append(None) yield lock.acquire_read() self.assertTrue(lock.locked()) phase.append(None) future = f() while len(phase) == 0: yield gen.Task(self.io_loop.add_callback) self.assertEqual(len(phase), 1) lock.release_read() yield future self.assertEqual(len(phase), 2) # Adapted from toro.test_lock.py class RWLockWriteTests(AsyncTestCase): @gen_test def test_acquire_release(self): lock = toro.RWLock(max_readers=10) self.assertFalse(lock.locked()) yield lock.acquire_write() self.assertTrue(lock.locked()) lock.release_write() self.assertFalse(lock.locked()) @gen_test def test_acquire_contended(self): lock = toro.RWLock(max_readers=10) yield lock.acquire_write() N = 5 @gen.coroutine def f(): yield lock.acquire_write() lock.release_write() futures = [f() for _ in range(N)] lock.release_write() yield futures @gen_test def test_reacquire(self): # Lock needs to be released before re-acquiring. lock = toro.RWLock(max_readers=10) phase = [] @gen.coroutine def f(): yield lock.acquire_write() self.assertTrue(lock.locked()) phase.append(None) yield lock.acquire_write() self.assertTrue(lock.locked()) phase.append(None) future = f() while len(phase) == 0: yield gen.Task(self.io_loop.add_callback) self.assertEqual(len(phase), 1) lock.release_write() yield future self.assertEqual(len(phase), 2) # Adapted from toro.test_lock.py class RWLockReadTests2(AsyncTestCase): def test_str(self): lock = toro.RWLock(max_readers=1) # No errors in various states. str(lock) lock.acquire_read() str(lock) @gen_test def test_acquire_timeout(self): lock = toro.RWLock(max_readers=1) self.assertTrue(lock.acquire_read().done()) self.assertTrue(lock.locked()) st = time.time() with assert_raises(toro.Timeout): yield lock.acquire_read(deadline=timedelta(seconds=0.1)) duration = time.time() - st self.assertAlmostEqual(0.1, duration, places=1) self.assertTrue(lock.locked()) @gen_test def test_acquire_callback(self): lock = toro.RWLock(max_readers=1) history = [] lock.acquire_read().add_done_callback(make_callback('acq1', history)) lock.acquire_read().add_done_callback(make_callback('acq2', history)) lock.release_read() history.append('release') self.assertEqual(['acq1', 'acq2', 'release'], history) def test_multi_release(self): lock = toro.RWLock(max_readers=1) lock.acquire_read() lock.release_read() self.assertRaises(RuntimeError, lock.release_read) # Adapted from toro.test_lock.py class RWLockWriteTests2(AsyncTestCase): def test_str(self): lock = toro.RWLock(max_readers=10) # No errors in various states. str(lock) lock.acquire_write() str(lock) @gen_test def test_acquire_timeout(self): lock = toro.RWLock(max_readers=10) yield lock.acquire_write() self.assertTrue(lock.locked()) st = time.time() with assert_raises(toro.Timeout): yield lock.acquire_write(deadline=timedelta(seconds=0.1)) duration = time.time() - st self.assertAlmostEqual(0.1, duration, places=1) self.assertTrue(lock.locked()) @gen_test def test_acquire_callback(self): lock = toro.RWLock(max_readers=10) history = [] lock.acquire_write().add_done_callback(make_callback('acq1', history)) future = lock.acquire_write() future.add_done_callback(make_callback('acq2', history)) lock.release_write() yield future history.append('release') self.assertEqual(['acq1', 'acq2', 'release'], history) def test_multi_release(self): lock = toro.RWLock(max_readers=10) lock.acquire_write() lock.release_write() self.assertRaises(RuntimeError, lock.release_write) class RWLockWithReadDefault(toro.RWLock): def acquire(self, *args, **kwargs): return self.acquire_read(*args, **kwargs) class RWLockWithWriteDefault(toro.RWLock): @gen.coroutine def acquire(self, *args, **kwargs): raise gen.Return((yield self.acquire_write(*args, **kwargs))) # Adapted from toro.test_lock.py class RWLockWithReadDefaultContextManagerTest(ContextManagerTestsMixin, AsyncTestCase): toro_class = RWLockWithReadDefault # Adapted from toro.test_lock.py class RWLockWithWriteDefaultContextManagerTest(ContextManagerTestsMixin, AsyncTestCase): toro_class = RWLockWithWriteDefault # Adapted from toro.test_lock.py class RWLockWithWriteMultipleReadersContextManagerTest(ContextManagerTestsMixin, AsyncTestCase): toro_class = partial(RWLockWithWriteDefault, max_readers=10) # Not adapted from toro lock tests, written just for RWLock class RWLockTests3(AsyncTestCase): @gen_test def test_read_lock(self): MAX_READERS = randint(2, 10) lock = toro.RWLock(max_readers=MAX_READERS) for i in range(MAX_READERS): self.assertFalse(lock.locked()) yield lock.acquire_read() self.assertTrue(lock.locked()) @gen_test def test_write_lock(self): MAX_READERS = randint(2, 10) lock = toro.RWLock(max_readers=MAX_READERS) self.assertFalse(lock.locked()) yield lock.acquire_write() self.assertTrue(lock.locked()) @gen_test def test_reacquire(self): MAX_READERS = randint(2, 10) # Lock needs to be released before re-acquiring. lock = toro.RWLock(max_readers=MAX_READERS) phase = [] @gen.coroutine def f(): yield lock.acquire_read() self.assertFalse(lock.locked()) phase.append(None) yield lock.acquire_write() self.assertTrue(lock.locked()) phase.append(None) future = f() while len(phase) == 0: yield gen.Task(self.io_loop.add_callback) self.assertEqual(len(phase), 1) lock.release_read() yield future self.assertEqual(len(phase), 2) @gen_test def test_reacquire_concurrent(self): MAX_READERS = 10 # Lock needs to be released before re-acquiring. lock = toro.RWLock(max_readers=MAX_READERS) phase = [] @gen.coroutine def f(): for _ in range(MAX_READERS): yield lock.acquire_read() self.assertTrue(lock.locked()) phase.append(None) # concurrent acquire_write yield [lock.acquire_write(), lock.acquire_write()] self.assertTrue(lock.locked()) phase.append(None) future = f() while len(phase) == 0: yield gen.Task(self.io_loop.add_callback) self.assertEqual(len(phase), 1) for i in range(MAX_READERS): lock.release_read() lock.release_write() yield future self.assertEqual(len(phase), 2) @gen_test def test_read_context_managers(self): MAX_READERS = 2 lock = toro.RWLock(max_readers=MAX_READERS) self.assertFalse(lock.locked()) with (yield lock.acquire_read()): self.assertFalse(lock.locked()) with (yield lock.acquire_read()): self.assertTrue(lock.locked()) self.assertFalse(lock.locked()) self.assertFalse(lock.locked()) @gen_test def test_write_context_managers(self): MAX_READERS = randint(2, 10) lock = toro.RWLock(max_readers=MAX_READERS) self.assertFalse(lock.locked()) with (yield lock.acquire_write()): self.assertTrue(lock.locked()) self.assertFalse(lock.locked()) @gen_test def test_write_acquire_timeout(self): MAX_READERS = randint(2, 10) lock = toro.RWLock(max_readers=MAX_READERS) yield lock.acquire_write() self.assertTrue(lock.locked()) st = time.time() with assert_raises(toro.Timeout): yield lock.acquire_write(deadline=timedelta(seconds=0.1)) duration = time.time() - st self.assertAlmostEqual(0.1, duration, places=1) self.assertTrue(lock.locked()) @gen_test def test_partial_release(self): lock = toro.RWLock(max_readers=5) N = 5 for i in range(N): yield lock.acquire_read() for i in range(N): lock.release_read() yield lock.acquire_write() lock.release_write() @gen_test def test_release_unlocked(self): lock = toro.RWLock(max_readers=5) N = 5 for i in range(N): yield lock.acquire_read() for i in range(N): lock.release_read() with assert_raises(RuntimeError): lock.release_read() toro-1.0.1/test/test_semaphore.py000066400000000000000000000151421265215761400170630ustar00rootroot00000000000000""" Test toro.Semaphore. Adapted from Gevent's lock_tests.py and test__semaphore.py. """ from datetime import timedelta import time import sys from tornado import gen from tornado.testing import gen_test, AsyncTestCase import toro from test import make_callback, assert_raises, ContextManagerTestsMixin # Adapted from Gevent's lock_tests.py class BaseSemaphoreTests(object): semtype = None def test_constructor(self): self.assertRaises(ValueError, self.semtype, value = -1) self.assertRaises(ValueError, self.semtype, value = -sys.maxint) def test_str(self): q = self.semtype(5) self.assertTrue(self.semtype.__name__ in str(q)) self.assertTrue('counter=5' in str(q)) @gen_test def test_acquire(self): sem = self.semtype(1) self.assertFalse(sem.locked()) result = sem.acquire() self.assertTrue(result) self.assertTrue(sem.locked()) # Wait for release() future = sem.wait() sem.release() yield future # Now wait() is instant yield sem.wait() sem = self.semtype(2) sem.acquire() sem.acquire() sem.release() sem.release() @gen_test def test_acquire_contended(self): sem = self.semtype(7) sem.acquire() N = 10 results1 = [] results2 = [] phase_num = 0 @gen.coroutine def f(): yield sem.acquire() results1.append(phase_num) yield sem.acquire() results2.append(phase_num) # Start independent tasks for i in range(N): f() # Let them all run until the counter reaches 0 while len(results1) + len(results2) < 6: yield gen.Task(self.io_loop.add_callback) self.assertEqual(results1 + results2, [0] * 6) phase_num = 1 for i in range(7): sem.release() while len(results1) + len(results2) < 13: yield gen.Task(self.io_loop.add_callback) self.assertEqual(sorted(results1 + results2), [0] * 6 + [1] * 7) phase_num = 2 for i in range(6): sem.release() while len(results1) + len(results2) < 19: yield gen.Task(self.io_loop.add_callback) self.assertEqual(sorted(results1 + results2), [0] * 6 + [1] * 7 + [2] * 6) # The semaphore is still locked self.assertTrue(sem.locked()) with assert_raises(toro.Timeout): yield sem.acquire(deadline=timedelta(seconds=0.1)) # Final release, to let the last task finish sem.release() @gen_test def test_try_acquire(self): sem = self.semtype(2) yield sem.acquire() yield sem.acquire() with assert_raises(toro.Timeout): yield sem.acquire(deadline=timedelta(seconds=0.1)) sem.release() yield sem.acquire() @gen_test def test_default_value(self): # The default initial value is 1. sem = self.semtype() sem.acquire() f_finished = [False] @gen.coroutine def f(): yield sem.acquire() # Allow switching yield gen.Task(self.io_loop.add_callback) sem.release() f_finished[0] = True future = f() # Let f run yield gen.Task(self.io_loop.add_timeout, time.time() + 0.01) self.assertFalse(f_finished[0]) sem.release() yield future @gen_test def test_wait(self): sem = self.semtype() sem.acquire() with assert_raises(toro.Timeout): yield sem.wait(deadline=timedelta(seconds=0.1)) sem.release() class SemaphoreTests(BaseSemaphoreTests, AsyncTestCase): """ Tests for unbounded semaphores. """ semtype = toro.Semaphore def test_release_unacquired(self): # Unbounded releases are allowed and increment the semaphore's value sem = self.semtype(1) sem.release() sem.acquire() sem.acquire() sem.release() class BoundedSemaphoreTests(BaseSemaphoreTests, AsyncTestCase): """ Tests for bounded semaphores. """ semtype = toro.BoundedSemaphore def test_release_unacquired(self): # Cannot go past the initial value sem = self.semtype() self.assertRaises(ValueError, sem.release) sem.acquire() sem.release() self.assertRaises(ValueError, sem.release) BoundedSemaphoreTests.__test__ = True # Adapted from Gevent's test__semaphore.py class TestTimeoutAcquire(AsyncTestCase): @gen_test def test_timeout_acquire(self): s = toro.Semaphore(value=0) with assert_raises(toro.Timeout): yield s.acquire(deadline=timedelta(seconds=0.01)) @gen_test def test_release_twice(self): s = toro.Semaphore() result = [] s.acquire().add_done_callback(lambda x: result.append('a')) s.release() s.acquire().add_done_callback(lambda x: result.append('b')) s.release() yield gen.Task(self.io_loop.add_timeout, time.time() + 0.01) self.assertEqual(result, ['a', 'b']) # Not adapted from Gevent's tests, specific to Toro class SemaphoreTests2(AsyncTestCase): def test_repr(self): # No exceptions str(toro.Semaphore()) repr(toro.Semaphore()) @gen_test def test_acquire_callback(self): # Test that callbacks passed to acquire() run immediately after # release(), and that wait() callbacks aren't run until a release() # with no waiters on acquire(). sem = toro.Semaphore(0) history = [] sem.acquire().add_done_callback(make_callback('acquire1', history)) sem.acquire().add_done_callback(make_callback('acquire2', history)) def wait_callback(name): def cb(_): self.assertFalse(sem.locked()) history.append(name) return cb sem.wait().add_done_callback(wait_callback('wait1')) sem.wait().add_done_callback(wait_callback('wait2')) sem.release() history.append('release1') sem.release() history.append('release2') sem.release() history.append('release3') self.assertEqual([ # First release wakes first acquire 'acquire1', 'release1', # Second release wakes second acquire 'acquire2', 'release2', # Third release wakes all waits 'wait1', 'wait2', 'release3' ], history) class SemaphoreContextManagerTest(ContextManagerTestsMixin, AsyncTestCase): toro_class = toro.Semaphore toro-1.0.1/test/test_timeout.py000066400000000000000000000003231265215761400165610ustar00rootroot00000000000000""" Test toro.Timeout exception. """ from tornado.testing import AsyncTestCase import toro class TestTimeout(AsyncTestCase): def test_str(self): exception = toro.Timeout() str(exception) toro-1.0.1/toro/000077500000000000000000000000001265215761400134705ustar00rootroot00000000000000toro-1.0.1/toro/__init__.py000066400000000000000000000745261265215761400156170ustar00rootroot00000000000000import contextlib import heapq import collections from functools import partial from Queue import Full, Empty import tornado from tornado import ioloop from tornado import gen from tornado.concurrent import Future version_tuple = (1, 0, 1) version = '.'.join(map(str, version_tuple)) """Current version of Toro.""" __all__ = [ # Exceptions 'NotReady', 'AlreadySet', 'Full', 'Empty', 'Timeout', # Primitives 'AsyncResult', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore', 'Lock', # Queues 'Queue', 'PriorityQueue', 'LifoQueue', 'JoinableQueue' ] class NotReady(Exception): """Raised when accessing an :class:`AsyncResult` that has no value yet.""" pass class AlreadySet(Exception): """Raised when setting a value on an :class:`AsyncResult` that already has one.""" pass class Timeout(Exception): """Raised when a deadline passes before a Future is ready.""" def __str__(self): return "Timeout" class _TimeoutFuture(Future): def __init__(self, deadline, io_loop): """Create a Future with optional deadline. If deadline is not None, it may be a number denoting a unix timestamp (as returned by ``io_loop.time()``) or a ``datetime.timedelta`` object for a deadline relative to the current time. set_exception(toro.Timeout()) is executed after a timeout. """ super(_TimeoutFuture, self).__init__() self.io_loop = io_loop if deadline is not None: callback = partial(self.set_exception, Timeout()) self._timeout_handle = io_loop.add_timeout(deadline, callback) else: self._timeout_handle = None def set_result(self, result): self._cancel_timeout() super(_TimeoutFuture, self).set_result(result) def set_exception(self, exception): self._cancel_timeout() super(_TimeoutFuture, self).set_exception(exception) def _cancel_timeout(self): if self._timeout_handle: self.io_loop.remove_timeout(self._timeout_handle) self._timeout_handle = None class _ContextManagerList(list): def __enter__(self, *args, **kwargs): for obj in self: obj.__enter__(*args, **kwargs) def __exit__(self, *args, **kwargs): for obj in self: obj.__exit__(*args, **kwargs) class _ContextManagerFuture(Future): """A Future that can be used with the "with" statement. When a coroutine yields this Future, the return value is a context manager that can be used like: with (yield future): pass At the end of the block, the Future's exit callback is run. Used for Lock.acquire, Semaphore.acquire, RWLock.acquire_read / acquire_write. """ def __init__(self, wrapped, exit_callback): super(_ContextManagerFuture, self).__init__() wrapped.add_done_callback(self._done_callback) self.exit_callback = exit_callback def _done_callback(self, wrapped): if wrapped.exception(): self.set_exception(wrapped.exception()) else: self.set_result(wrapped.result()) def result(self): if self.exception(): raise self.exception() # Otherwise return a context manager that cleans up after the block. @contextlib.contextmanager def f(): try: yield finally: self.exit_callback() return f() def _consume_expired_waiters(waiters): # Delete waiters at the head of the queue who've timed out while waiters and waiters[0].done(): waiters.popleft() _null_result = object() class AsyncResult(object): """A one-time event that stores a value or an exception. The only distinction between AsyncResult and a simple Future is that AsyncResult lets coroutines wait with a deadline. The deadline can be configured separately for each waiter. An :class:`AsyncResult` instance cannot be reset. :Parameters: - `io_loop`: Optional custom IOLoop. """ def __init__(self, io_loop=None): self.io_loop = io_loop or ioloop.IOLoop.current() self.value = _null_result self.waiters = [] def __str__(self): result = '<%s ' % (self.__class__.__name__, ) if self.ready(): result += 'value=%r' % self.value else: result += 'unset' if self.waiters: result += ' waiters[%s]' % len(self.waiters) return result + '>' def set(self, value): """Set a value and wake up all the waiters.""" if self.ready(): raise AlreadySet self.value = value waiters, self.waiters = self.waiters, [] for waiter in waiters: if not waiter.done(): # Might have timed out waiter.set_result(value) def ready(self): return self.value is not _null_result def get(self, deadline=None): """Get a value once :meth:`set` is called. Returns a Future. The Future's result will be the value. The Future raises :exc:`toro.Timeout` if no value is set before the deadline. :Parameters: - `deadline`: Optional timeout, either an absolute timestamp (as returned by ``io_loop.time()``) or a ``datetime.timedelta`` for a deadline relative to the current time. """ future = _TimeoutFuture(deadline, self.io_loop) if self.ready(): future.set_result(self.value) else: self.waiters.append(future) return future def get_nowait(self): """Get the value if ready, or raise :class:`NotReady`.""" if self.ready(): return self.value else: raise NotReady class Condition(object): """A condition allows one or more coroutines to wait until notified. Like a standard Condition_, but does not need an underlying lock that is acquired and released. .. _Condition: http://docs.python.org/library/threading.html#threading.Condition :Parameters: - `io_loop`: Optional custom IOLoop. """ def __init__(self, io_loop=None): self.io_loop = io_loop or ioloop.IOLoop.current() self.waiters = collections.deque() # Queue of _Waiter objects def __str__(self): result = '<%s' % (self.__class__.__name__, ) if self.waiters: result += ' waiters[%s]' % len(self.waiters) return result + '>' def wait(self, deadline=None): """Wait for :meth:`notify`. Returns a Future. :exc:`~toro.Timeout` is executed after a timeout. :Parameters: - `deadline`: Optional timeout, either an absolute timestamp (as returned by ``io_loop.time()``) or a ``datetime.timedelta`` for a deadline relative to the current time. """ future = _TimeoutFuture(deadline, self.io_loop) self.waiters.append(future) return future def notify(self, n=1): """Wake up `n` waiters. :Parameters: - `n`: The number of waiters to awaken (default: 1) """ waiters = [] # Waiters we plan to run right now while n and self.waiters: waiter = self.waiters.popleft() if not waiter.done(): # Might have timed out n -= 1 waiters.append(waiter) for waiter in waiters: waiter.set_result(None) def notify_all(self): """Wake up all waiters.""" self.notify(len(self.waiters)) # TODO: show correct examples that avoid thread / process issues w/ concurrent.futures.Future class Event(object): """An event blocks coroutines until its internal flag is set to True. Similar to threading.Event_. .. _threading.Event: http://docs.python.org/library/threading.html#threading.Event .. seealso:: :doc:`examples/event_example` :Parameters: - `io_loop`: Optional custom IOLoop. """ def __init__(self, io_loop=None): self.io_loop = io_loop or ioloop.IOLoop.current() self.condition = Condition(io_loop=io_loop) self._flag = False def __str__(self): return '<%s %s>' % ( self.__class__.__name__, 'set' if self._flag else 'clear') def is_set(self): """Return ``True`` if and only if the internal flag is true.""" return self._flag def set(self): """Set the internal flag to ``True``. All waiters are awakened. Calling :meth:`wait` once the flag is true will not block. """ self._flag = True self.condition.notify_all() def clear(self): """Reset the internal flag to ``False``. Calls to :meth:`wait` will block until :meth:`set` is called. """ self._flag = False def wait(self, deadline=None): """Block until the internal flag is true. Returns a Future. The Future raises :exc:`~toro.Timeout` after a timeout. :Parameters: - `callback`: Function taking no arguments. - `deadline`: Optional timeout, either an absolute timestamp (as returned by ``io_loop.time()``) or a ``datetime.timedelta`` for a deadline relative to the current time. """ if self._flag: future = _TimeoutFuture(None, self.io_loop) future.set_result(None) return future else: return self.condition.wait(deadline) class Queue(object): """Create a queue object with a given maximum size. If `maxsize` is 0 (the default) the queue size is unbounded. Unlike the `standard Queue`_, you can reliably know this Queue's size with :meth:`qsize`, since your single-threaded Tornado application won't be interrupted between calling :meth:`qsize` and doing an operation on the Queue. **Examples:** :doc:`examples/producer_consumer_example` :doc:`examples/web_spider_example` :Parameters: - `maxsize`: Optional size limit (no limit by default). - `io_loop`: Optional custom IOLoop. .. _`Gevent's Queue`: http://www.gevent.org/gevent.queue.html .. _`standard Queue`: http://docs.python.org/library/queue.html#Queue.Queue """ def __init__(self, maxsize=0, io_loop=None): self.io_loop = io_loop or ioloop.IOLoop.current() if maxsize is None: raise TypeError("maxsize can't be None") if maxsize < 0: raise ValueError("maxsize can't be negative") self._maxsize = maxsize # _TimeoutFutures self.getters = collections.deque([]) # Pairs of (item, _TimeoutFuture) self.putters = collections.deque([]) self._init(maxsize) # These three are overridable in subclasses. def _init(self, maxsize): self.queue = collections.deque() def _get(self): return self.queue.popleft() def _put(self, item): self.queue.append(item) def __repr__(self): return '<%s at %s %s>' % ( type(self).__name__, hex(id(self)), self._format()) def __str__(self): return '<%s %s>' % (type(self).__name__, self._format()) def _format(self): result = 'maxsize=%r' % (self.maxsize, ) if getattr(self, 'queue', None): result += ' queue=%r' % self.queue if self.getters: result += ' getters[%s]' % len(self.getters) if self.putters: result += ' putters[%s]' % len(self.putters) return result def _consume_expired_putters(self): # Delete waiters at the head of the queue who've timed out while self.putters and self.putters[0][1].done(): self.putters.popleft() def qsize(self): """Number of items in the queue""" return len(self.queue) @property def maxsize(self): """Number of items allowed in the queue.""" return self._maxsize def empty(self): """Return ``True`` if the queue is empty, ``False`` otherwise.""" return not self.queue def full(self): """Return ``True`` if there are `maxsize` items in the queue. .. note:: if the Queue was initialized with `maxsize=0` (the default), then :meth:`full` is never ``True``. """ if self.maxsize == 0: return False else: return self.maxsize <= self.qsize() def put(self, item, deadline=None): """Put an item into the queue. Returns a Future. The Future blocks until a free slot is available for `item`, or raises :exc:`toro.Timeout`. :Parameters: - `deadline`: Optional timeout, either an absolute timestamp (as returned by ``io_loop.time()``) or a ``datetime.timedelta`` for a deadline relative to the current time. """ _consume_expired_waiters(self.getters) future = _TimeoutFuture(deadline, self.io_loop) if self.getters: assert not self.queue, "queue non-empty, why are getters waiting?" getter = self.getters.popleft() # Use _put and _get instead of passing item straight to getter, in # case a subclass has logic that must run (e.g. JoinableQueue). self._put(item) getter.set_result(self._get()) future.set_result(None) else: if self.maxsize and self.maxsize <= self.qsize(): self.putters.append((item, future)) else: self._put(item) future.set_result(None) return future def put_nowait(self, item): """Put an item into the queue without blocking. If no free slot is immediately available, raise queue.Full. """ _consume_expired_waiters(self.getters) if self.getters: assert not self.queue, "queue non-empty, why are getters waiting?" getter = self.getters.popleft() self._put(item) getter.set_result(self._get()) elif self.maxsize and self.maxsize <= self.qsize(): raise Full else: self._put(item) def get(self, deadline=None): """Remove and return an item from the queue. Returns a Future. The Future blocks until an item is available, or raises :exc:`toro.Timeout`. :Parameters: - `deadline`: Optional timeout, either an absolute timestamp (as returned by ``io_loop.time()``) or a ``datetime.timedelta`` for a deadline relative to the current time. """ self._consume_expired_putters() future = _TimeoutFuture(deadline, self.io_loop) if self.putters: assert self.full(), "queue not full, why are putters waiting?" item, putter = self.putters.popleft() self._put(item) putter.set_result(None) future.set_result(self._get()) elif self.qsize(): future.set_result(self._get()) else: self.getters.append(future) return future def get_nowait(self): """Remove and return an item from the queue without blocking. Return an item if one is immediately available, else raise :exc:`queue.Empty`. """ self._consume_expired_putters() if self.putters: assert self.full(), "queue not full, why are putters waiting?" item, putter = self.putters.popleft() self._put(item) putter.set_result(None) return self._get() elif self.qsize(): return self._get() else: raise Empty class PriorityQueue(Queue): """A subclass of :class:`Queue` that retrieves entries in priority order (lowest first). Entries are typically tuples of the form: ``(priority number, data)``. :Parameters: - `maxsize`: Optional size limit (no limit by default). - `initial`: Optional sequence of initial items. - `io_loop`: Optional custom IOLoop. """ def _init(self, maxsize): self.queue = [] def _put(self, item, heappush=heapq.heappush): heappush(self.queue, item) def _get(self, heappop=heapq.heappop): return heappop(self.queue) class LifoQueue(Queue): """A subclass of :class:`Queue` that retrieves most recently added entries first. :Parameters: - `maxsize`: Optional size limit (no limit by default). - `initial`: Optional sequence of initial items. - `io_loop`: Optional custom IOLoop. """ def _init(self, maxsize): self.queue = [] def _put(self, item): self.queue.append(item) def _get(self): return self.queue.pop() class JoinableQueue(Queue): """A subclass of :class:`Queue` that additionally has :meth:`task_done` and :meth:`join` methods. .. seealso:: :doc:`examples/web_spider_example` :Parameters: - `maxsize`: Optional size limit (no limit by default). - `initial`: Optional sequence of initial items. - `io_loop`: Optional custom IOLoop. """ def __init__(self, maxsize=0, io_loop=None): Queue.__init__(self, maxsize=maxsize, io_loop=io_loop) self.unfinished_tasks = 0 self._finished = Event(io_loop) self._finished.set() def _format(self): result = Queue._format(self) if self.unfinished_tasks: result += ' tasks=%s' % self.unfinished_tasks return result def _put(self, item): self.unfinished_tasks += 1 self._finished.clear() Queue._put(self, item) def task_done(self): """Indicate that a formerly enqueued task is complete. Used by queue consumers. For each :meth:`get ` used to fetch a task, a subsequent call to :meth:`task_done` tells the queue that the processing on the task is complete. If a :meth:`join` is currently blocking, it will resume when all items have been processed (meaning that a :meth:`task_done` call was received for every item that had been :meth:`put ` into the queue). Raises ``ValueError`` if called more times than there were items placed in the queue. """ if self.unfinished_tasks <= 0: raise ValueError('task_done() called too many times') self.unfinished_tasks -= 1 if self.unfinished_tasks == 0: self._finished.set() def join(self, deadline=None): """Block until all items in the queue are processed. Returns a Future. The count of unfinished tasks goes up whenever an item is added to the queue. The count goes down whenever a consumer calls :meth:`task_done` to indicate that all work on the item is complete. When the count of unfinished tasks drops to zero, :meth:`join` unblocks. The Future raises :exc:`toro.Timeout` if the count is not zero before the deadline. :Parameters: - `deadline`: Optional timeout, either an absolute timestamp (as returned by ``io_loop.time()``) or a ``datetime.timedelta`` for a deadline relative to the current time. """ return self._finished.wait(deadline) class Semaphore(object): """A lock that can be acquired a fixed number of times before blocking. A Semaphore manages a counter representing the number of release() calls minus the number of acquire() calls, plus an initial value. The acquire() method blocks if necessary until it can return without making the counter negative. If not given, value defaults to 1. :meth:`acquire` supports the context manager protocol: >>> from tornado import gen >>> import toro >>> semaphore = toro.Semaphore() >>> >>> @gen.coroutine ... def f(): ... with (yield semaphore.acquire()): ... assert semaphore.locked() ... ... assert not semaphore.locked() .. note:: Unlike the standard threading.Semaphore_, a :class:`Semaphore` can tell you the current value of its :attr:`counter`, because code in a single-threaded Tornado app can check these values and act upon them without fear of interruption from another thread. .. _threading.Semaphore: http://docs.python.org/library/threading.html#threading.Semaphore .. seealso:: :doc:`examples/web_spider_example` :Parameters: - `value`: An int, the initial value (default 1). - `io_loop`: Optional custom IOLoop. """ def __init__(self, value=1, io_loop=None): if value < 0: raise ValueError('semaphore initial value must be >= 0') # The semaphore is implemented as a Queue with 'value' objects self.q = Queue(io_loop=io_loop) for _ in range(value): self.q.put_nowait(None) self._unlocked = Event(io_loop=io_loop) if value: self._unlocked.set() def __repr__(self): return '<%s at %s%s>' % ( type(self).__name__, hex(id(self)), self._format()) def __str__(self): return '<%s%s>' % ( self.__class__.__name__, self._format()) def _format(self): return ' counter=%s' % self.counter @property def counter(self): """An integer, the current semaphore value""" return self.q.qsize() def locked(self): """True if :attr:`counter` is zero""" return self.q.empty() def release(self): """Increment :attr:`counter` and wake one waiter. """ self.q.put(None) if not self.locked(): # No one was waiting on acquire(), so self.q.qsize() is positive self._unlocked.set() def wait(self, deadline=None): """Wait for :attr:`locked` to be False. Returns a Future. The Future raises :exc:`toro.Timeout` after the deadline. :Parameters: - `deadline`: Optional timeout, either an absolute timestamp (as returned by ``io_loop.time()``) or a ``datetime.timedelta`` for a deadline relative to the current time. """ return self._unlocked.wait(deadline) def acquire(self, deadline=None): """Decrement :attr:`counter`. Returns a Future. Block if the counter is zero and wait for a :meth:`release`. The Future raises :exc:`toro.Timeout` after the deadline. :Parameters: - `deadline`: Optional timeout, either an absolute timestamp (as returned by ``io_loop.time()``) or a ``datetime.timedelta`` for a deadline relative to the current time. """ queue_future = self.q.get(deadline) if self.q.empty(): self._unlocked.clear() future = _ContextManagerFuture(queue_future, self.release) return future def __enter__(self): raise RuntimeError( "Use Semaphore like 'with (yield semaphore)', not like" " 'with semaphore'") __exit__ = __enter__ class BoundedSemaphore(Semaphore): """A semaphore that prevents release() being called too often. A bounded semaphore checks to make sure its current value doesn't exceed its initial value. If it does, ``ValueError`` is raised. In most situations semaphores are used to guard resources with limited capacity. If the semaphore is released too many times it's a sign of a bug. If not given, *value* defaults to 1. .. seealso:: :doc:`examples/web_spider_example` """ def __init__(self, value=1, io_loop=None): super(BoundedSemaphore, self).__init__(value=value, io_loop=io_loop) self._initial_value = value def release(self): if self.counter >= self._initial_value: raise ValueError("Semaphore released too many times") return super(BoundedSemaphore, self).release() class Lock(object): """A lock for coroutines. It is created unlocked. When unlocked, :meth:`acquire` changes the state to locked. When the state is locked, yielding :meth:`acquire` waits until a call to :meth:`release`. The :meth:`release` method should only be called in the locked state; an attempt to release an unlocked lock raises RuntimeError. When more than one coroutine is waiting for the lock, the first one registered is awakened by :meth:`release`. :meth:`acquire` supports the context manager protocol: >>> from tornado import gen >>> import toro >>> lock = toro.Lock() >>> >>> @gen.coroutine ... def f(): ... with (yield lock.acquire()): ... assert lock.locked() ... ... assert not lock.locked() .. note:: Unlike with the standard threading.Lock_, code in a single-threaded Tornado application can check if a :class:`Lock` is :meth:`locked`, and act on that information without fear that another thread has grabbed the lock, provided you do not yield to the IOLoop between checking :meth:`locked` and using a protected resource. .. _threading.Lock: http://docs.python.org/2/library/threading.html#lock-objects .. seealso:: :doc:`examples/lock_example` :Parameters: - `io_loop`: Optional custom IOLoop. """ def __init__(self, io_loop=None): self._block = BoundedSemaphore(value=1, io_loop=io_loop) def __str__(self): return "<%s _block=%s>" % ( self.__class__.__name__, self._block) def acquire(self, deadline=None): """Attempt to lock. Returns a Future. The Future raises :exc:`toro.Timeout` if the deadline passes. :Parameters: - `deadline`: Optional timeout, either an absolute timestamp (as returned by ``io_loop.time()``) or a ``datetime.timedelta`` for a deadline relative to the current time. """ return self._block.acquire(deadline) def release(self): """Unlock. If any coroutines are waiting for :meth:`acquire`, the first in line is awakened. If not locked, raise a RuntimeError. """ if not self.locked(): raise RuntimeError('release unlocked lock') self._block.release() def locked(self): """``True`` if the lock has been acquired""" return self._block.locked() def __enter__(self): raise RuntimeError( "Use Lock like 'with (yield lock)', not like" " 'with lock'") __exit__ = __enter__ if tornado.version_info[:2] >= (4, 2): tornado_multi_future = gen.multi_future else: tornado_multi_future = lambda futures, quiet_exceptions: futures class RWLock(object): """A reader-writer lock for coroutines. It is created unlocked. When unlocked, :meth:`acquire_write` always changes the state to locked. When unlocked, :meth:`acquire_read` can changed the state to locked, if :meth:`acquire_read` was called max_readers times. When the state is locked, yielding :meth:`acquire_read`/meth:`acquire_write` waits until a call to :meth:`release_write` in case of locking on write, or :meth:`release_read` in case of locking on read. The :meth:`release_read` method should only be called in the locked-on-read state; an attempt to release an unlocked lock raises RuntimeError. The :meth:`release_write` method should only be called in the locked on write state; an attempt to release an unlocked lock raises RuntimeError. When more than one coroutine is waiting for the lock, the first one registered is awakened by :meth:`release_read`/:meth:`release_write`. :meth:`acquire_read`/:meth:`acquire_write` support the context manager protocol: >>> from tornado import gen >>> import toro >>> lock = toro.RWLock(max_readers=10) >>> >>> @gen.coroutine ... def f(): ... with (yield lock.acquire_read()): ... assert not lock.locked() ... ... with (yield lock.acquire_write()): ... assert lock.locked() ... ... assert not lock.locked() :Parameters: - `max_readers`: Optional max readers value, default 1. - `io_loop`: Optional custom IOLoop. """ def __init__(self, max_readers=1, io_loop=None): self._max_readers = max_readers self._block = BoundedSemaphore(value=max_readers, io_loop=io_loop) def __str__(self): return "<%s _block=%s>" % ( self.__class__.__name__, self._block) def acquire_read(self, deadline=None): """Attempt to lock for read. Returns a Future. The Future raises :exc:`toro.Timeout` if the deadline passes. :Parameters: - `deadline`: Optional timeout, either an absolute timestamp (as returned by ``io_loop.time()``) or a ``datetime.timedelta`` for a deadline relative to the current time. """ return self._block.acquire(deadline) @gen.coroutine def acquire_write(self, deadline=None): """Attempt to lock for write. Returns a Future. The Future raises :exc:`toro.Timeout` if the deadline passes. :Parameters: - `deadline`: Optional timeout, either an absolute timestamp (as returned by ``io_loop.time()``) or a ``datetime.timedelta`` for a deadline relative to the current time. """ futures = [self._block.acquire(deadline) for _ in xrange(self._max_readers)] try: managers = yield tornado_multi_future(futures, quiet_exceptions=Timeout) except Timeout: for f in futures: # Avoid traceback logging. f.exception() raise raise gen.Return(_ContextManagerList(managers)) def release_read(self): """Releases one reader. If any coroutines are waiting for :meth:`acquire_read` (in case of full readers queue), the first in line is awakened. If not locked, raise a RuntimeError. """ if self._block.counter == self._max_readers: raise RuntimeError('release unlocked lock') self._block.release() def release_write(self): """Releases after write. The first in queue will be awakened after release. If not locked, raise a RuntimeError. """ if not self.locked(): raise RuntimeError('release unlocked lock') for i in xrange(self._max_readers): self._block.release() def locked(self): """``True`` if the lock has been acquired""" return self._block.locked() def __enter__(self): raise RuntimeError( "Use RWLock like 'with (yield lock)', not like" " 'with lock'") __exit__ = __enter__ toro-1.0.1/tox.ini000066400000000000000000000006621265215761400140240ustar00rootroot00000000000000# Tox (http://tox.testrun.org/) is a tool for running tests in multiple # virtualenvs. This configuration file will run the test suite on all supported # python versions. To use it, "pip install tox" and then run "tox" from this # directory. [tox] envlist = py26, py27, py32, py33, py34, pypy [testenv] commands = {envpython} setup.py test [testenv:docs] commands = pip install sphinx sphinx-build -b doctest doc build