3 Athana - standalone web server including the TAL template language
5 Copyright (C) 2007 Matthias Kramm <kramm@in.tum.de>
7 This program is free software: you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation, either version 3 of the License, or
10 (at your option) any later version.
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with this program. If not, see <http://www.gnu.org/licenses/>.
21 #===============================================================
25 # A standalone webserver based on Medusa and the Zope TAL Parser
27 # This file is distributed under the GPL, see file COPYING for details.
29 #===============================================================
31 Parse HTML and compile to TALInterpreter intermediate code.
34 RCS_ID = '$Id: athana.py,v 1.15 2007/11/23 10:13:32 kramm Exp $'
38 from HTMLParser import HTMLParser, HTMLParseError
40 BOOLEAN_HTML_ATTRS = [
41 "compact", "nowrap", "ismap", "declare", "noshade", "checked",
42 "disabled", "readonly", "multiple", "selected", "noresize",
47 "base", "meta", "link", "hr", "br", "param", "img", "area",
48 "input", "col", "basefont", "isindex", "frame",
51 PARA_LEVEL_HTML_TAGS = [
52 "h1", "h2", "h3", "h4", "h5", "h6", "p",
55 BLOCK_CLOSING_TAG_MAP = {
56 "tr": ("tr", "td", "th"),
64 BLOCK_LEVEL_HTML_TAGS = [
65 "blockquote", "table", "tr", "th", "td", "thead", "tfoot", "tbody",
66 "noframe", "ul", "ol", "li", "dl", "dt", "dd", "div",
69 TIGHTEN_IMPLICIT_CLOSE_TAGS = (PARA_LEVEL_HTML_TAGS
70 + BLOCK_CLOSING_TAG_MAP.keys())
73 class NestingError(HTMLParseError):
74 """Exception raised when elements aren't properly nested."""
76 def __init__(self, tagstack, endtag, position=(None, None)):
79 if len(tagstack) == 1:
80 msg = ('Open tag <%s> does not match close tag </%s>'
81 % (tagstack[0], endtag))
83 msg = ('Open tags <%s> do not match close tag </%s>'
84 % ('>, <'.join(tagstack), endtag))
86 msg = 'No tags are open to match </%s>' % endtag
87 HTMLParseError.__init__(self, msg, position)
89 class EmptyTagError(NestingError):
90 """Exception raised when empty elements have an end tag."""
92 def __init__(self, tag, position=(None, None)):
94 msg = 'Close tag </%s> should be removed' % tag
95 HTMLParseError.__init__(self, msg, position)
97 class OpenTagError(NestingError):
98 """Exception raised when a tag is not allowed in another tag."""
100 def __init__(self, tagstack, tag, position=(None, None)):
102 msg = 'Tag <%s> is not allowed in <%s>' % (tag, tagstack[-1])
103 HTMLParseError.__init__(self, msg, position)
105 class HTMLTALParser(HTMLParser):
108 def __init__(self, gen=None):
109 HTMLParser.__init__(self)
111 gen = TALGenerator(xml=0)
115 self.nsdict = {'tal': ZOPE_TAL_NS,
116 'metal': ZOPE_METAL_NS,
117 'i18n': ZOPE_I18N_NS,
120 def parseFile(self, file):
125 self.parseString(data)
130 def parseString(self, data):
134 self.implied_endtag(self.tagstack[-1], 2)
135 assert self.nsstack == [], self.nsstack
138 return self.gen.getCode()
140 def getWarnings(self):
144 def handle_starttag(self, tag, attrs):
145 self.close_para_tags(tag)
146 self.scan_xmlns(attrs)
147 tag, attrlist, taldict, metaldict, i18ndict \
148 = self.process_ns(tag, attrs)
149 if tag in EMPTY_HTML_TAGS and taldict.get("content"):
151 "empty HTML tags cannot use tal:content: %s" % `tag`,
153 self.tagstack.append(tag)
154 self.gen.emitStartElement(tag, attrlist, taldict, metaldict, i18ndict,
156 if tag in EMPTY_HTML_TAGS:
157 self.implied_endtag(tag, -1)
159 def handle_startendtag(self, tag, attrs):
160 self.close_para_tags(tag)
161 self.scan_xmlns(attrs)
162 tag, attrlist, taldict, metaldict, i18ndict \
163 = self.process_ns(tag, attrs)
164 if taldict.get("content"):
165 if tag in EMPTY_HTML_TAGS:
167 "empty HTML tags cannot use tal:content: %s" % `tag`,
169 self.gen.emitStartElement(tag, attrlist, taldict, metaldict,
170 i18ndict, self.getpos())
171 self.gen.emitEndElement(tag, implied=-1)
173 self.gen.emitStartElement(tag, attrlist, taldict, metaldict,
174 i18ndict, self.getpos(), isend=1)
177 def handle_endtag(self, tag):
178 if tag in EMPTY_HTML_TAGS:
179 raise EmptyTagError(tag, self.getpos())
180 self.close_enclosed_tags(tag)
181 self.gen.emitEndElement(tag)
185 def close_para_tags(self, tag):
186 if tag in EMPTY_HTML_TAGS:
189 if BLOCK_CLOSING_TAG_MAP.has_key(tag):
190 blocks_to_close = BLOCK_CLOSING_TAG_MAP[tag]
191 for i in range(len(self.tagstack)):
193 if t in blocks_to_close:
196 elif t in BLOCK_LEVEL_HTML_TAGS:
198 elif tag in PARA_LEVEL_HTML_TAGS + BLOCK_LEVEL_HTML_TAGS:
199 i = len(self.tagstack) - 1
201 closetag = self.tagstack[i]
202 if closetag in BLOCK_LEVEL_HTML_TAGS:
204 if closetag in PARA_LEVEL_HTML_TAGS:
206 raise OpenTagError(self.tagstack, tag, self.getpos())
210 while len(self.tagstack) > close_to:
211 self.implied_endtag(self.tagstack[-1], 1)
213 def close_enclosed_tags(self, tag):
214 if tag not in self.tagstack:
215 raise NestingError(self.tagstack, tag, self.getpos())
216 while tag != self.tagstack[-1]:
217 self.implied_endtag(self.tagstack[-1], 1)
218 assert self.tagstack[-1] == tag
220 def implied_endtag(self, tag, implied):
221 assert tag == self.tagstack[-1]
222 assert implied in (-1, 1, 2)
223 isend = (implied < 0)
224 if tag in TIGHTEN_IMPLICIT_CLOSE_TAGS:
225 white = self.gen.unEmitWhitespace()
228 self.gen.emitEndElement(tag, isend=isend, implied=implied)
230 self.gen.emitRawText(white)
234 def handle_charref(self, name):
235 self.gen.emitRawText("&#%s;" % name)
237 def handle_entityref(self, name):
238 self.gen.emitRawText("&%s;" % name)
240 def handle_data(self, data):
241 self.gen.emitRawText(data)
243 def handle_comment(self, data):
244 self.gen.emitRawText("<!--%s-->" % data)
246 def handle_decl(self, data):
247 self.gen.emitRawText("<!%s>" % data)
249 def handle_pi(self, data):
250 self.gen.emitRawText("<?%s>" % data)
253 def scan_xmlns(self, attrs):
255 for key, value in attrs:
256 if key.startswith("xmlns:"):
257 nsnew[key[6:]] = value
259 self.nsstack.append(self.nsdict)
260 self.nsdict = self.nsdict.copy()
261 self.nsdict.update(nsnew)
263 self.nsstack.append(self.nsdict)
266 self.nsdict = self.nsstack.pop()
268 def fixname(self, name):
270 prefix, suffix = name.split(':', 1)
271 if prefix == 'xmlns':
272 nsuri = self.nsdict.get(suffix)
273 if nsuri in (ZOPE_TAL_NS, ZOPE_METAL_NS, ZOPE_I18N_NS):
274 return name, name, prefix
276 nsuri = self.nsdict.get(prefix)
277 if nsuri == ZOPE_TAL_NS:
278 return name, suffix, 'tal'
279 elif nsuri == ZOPE_METAL_NS:
280 return name, suffix, 'metal'
281 elif nsuri == ZOPE_I18N_NS:
282 return name, suffix, 'i18n'
285 def process_ns(self, name, attrs):
290 name, namebase, namens = self.fixname(name)
293 key, keybase, keyns = self.fixname(key)
294 ns = keyns or namens # default to tag namespace
295 if ns and ns != 'unknown':
296 item = (key, value, ns)
298 if taldict.has_key(keybase):
299 raise TALError("duplicate TAL attribute " +
300 `keybase`, self.getpos())
301 taldict[keybase] = value
303 if metaldict.has_key(keybase):
304 raise METALError("duplicate METAL attribute " +
305 `keybase`, self.getpos())
306 metaldict[keybase] = value
308 if i18ndict.has_key(keybase):
309 raise I18NError("duplicate i18n attribute " +
310 `keybase`, self.getpos())
311 i18ndict[keybase] = value
312 attrlist.append(item)
313 if namens in ('metal', 'tal'):
314 taldict['tal tag'] = namens
315 return name, attrlist, taldict, metaldict, i18ndict
317 Generic expat-based XML parser base class.
323 ordered_attributes = 0
326 "StartElementHandler",
328 "ProcessingInstructionHandler",
329 "CharacterDataHandler",
330 "UnparsedEntityDeclHandler",
331 "NotationDeclHandler",
332 "StartNamespaceDeclHandler",
333 "EndNamespaceDeclHandler",
335 "StartCdataSectionHandler",
336 "EndCdataSectionHandler",
338 "DefaultHandlerExpand",
339 "NotStandaloneHandler",
340 "ExternalEntityRefHandler",
342 "StartDoctypeDeclHandler",
343 "EndDoctypeDeclHandler",
344 "ElementDeclHandler",
348 def __init__(self, encoding=None):
349 self.parser = p = self.createParser()
350 if self.ordered_attributes:
352 self.parser.ordered_attributes = self.ordered_attributes
353 except AttributeError:
354 print "Can't set ordered_attributes"
355 self.ordered_attributes = 0
356 for name in self.handler_names:
357 method = getattr(self, name, None)
358 if method is not None:
360 setattr(p, name, method)
361 except AttributeError:
362 print "Can't set expat handler %s" % name
364 def createParser(self, encoding=None):
367 from Products.ParsedXML.Expat import pyexpat
368 XMLParseError = pyexpat.ExpatError
369 return pyexpat.ParserCreate(encoding, ' ')
371 from xml.parsers import expat
372 XMLParseError = expat.ExpatError
373 return expat.ParserCreate(encoding, ' ')
375 def parseFile(self, filename):
378 #self.parseStream(open(filename))
380 def parseString(self, s):
381 self.parser.Parse(s, 1)
383 def parseURL(self, url):
385 self.parseStream(urllib.urlopen(url))
387 def parseStream(self, stream):
388 self.parser.ParseFile(stream)
390 def parseFragment(self, s, end=0):
391 self.parser.Parse(s, end)
392 """Interface that a TALES engine provides to the METAL/TAL implementation."""
395 from Interface import Interface
396 from Interface.Attribute import Attribute
398 class Interface: pass
399 def Attribute(*args): pass
402 class ITALESCompiler(Interface):
403 """Compile-time interface provided by a TALES implementation.
405 The TAL compiler needs an instance of this interface to support
406 compilation of TALES expressions embedded in documents containing
407 TAL and METAL constructs.
410 def getCompilerError():
411 """Return the exception class raised for compilation errors.
414 def compile(expression):
415 """Return a compiled form of 'expression' for later evaluation.
417 'expression' is the source text of the expression.
419 The return value may be passed to the various evaluate*()
420 methods of the ITALESEngine interface. No compatibility is
421 required for the values of the compiled expression between
422 different ITALESEngine implementations.
426 class ITALESEngine(Interface):
427 """Render-time interface provided by a TALES implementation.
429 The TAL interpreter uses this interface to TALES to support
430 evaluation of the compiled expressions returned by
431 ITALESCompiler.compile().
435 """Return an object that supports ITALESCompiler."""
438 """Return the value of the 'default' TALES expression.
440 Checking a value for a match with 'default' should be done
441 using the 'is' operator in Python.
444 def setPosition((lineno, offset)):
445 """Inform the engine of the current position in the source file.
447 This is used to allow the evaluation engine to report
448 execution errors so that site developers can more easily
449 locate the offending expression.
452 def setSourceFile(filename):
453 """Inform the engine of the name of the current source file.
455 This is used to allow the evaluation engine to report
456 execution errors so that site developers can more easily
457 locate the offending expression.
461 """Push a new scope onto the stack of open scopes.
465 """Pop one scope from the stack of open scopes.
468 def evaluate(compiled_expression):
469 """Evaluate an arbitrary expression.
471 No constraints are imposed on the return value.
474 def evaluateBoolean(compiled_expression):
475 """Evaluate an expression that must return a Boolean value.
478 def evaluateMacro(compiled_expression):
479 """Evaluate an expression that must return a macro program.
482 def evaluateStructure(compiled_expression):
483 """Evaluate an expression that must return a structured
486 The result of evaluating 'compiled_expression' must be a
487 string containing a parsable HTML or XML fragment. Any TAL
488 markup cnotained in the result string will be interpreted.
491 def evaluateText(compiled_expression):
492 """Evaluate an expression that must return text.
494 The returned text should be suitable for direct inclusion in
495 the output: any HTML or XML escaping or quoting is the
496 responsibility of the expression itself.
499 def evaluateValue(compiled_expression):
500 """Evaluate an arbitrary expression.
502 No constraints are imposed on the return value.
505 def createErrorInfo(exception, (lineno, offset)):
506 """Returns an ITALESErrorInfo object.
508 The returned object is used to provide information about the
509 error condition for the on-error handler.
512 def setGlobal(name, value):
513 """Set a global variable.
515 The variable will be named 'name' and have the value 'value'.
518 def setLocal(name, value):
519 """Set a local variable in the current scope.
521 The variable will be named 'name' and have the value 'value'.
524 def setRepeat(name, compiled_expression):
528 def translate(domain, msgid, mapping, default=None):
530 See ITranslationService.translate()
534 class ITALESErrorInfo(Interface):
536 type = Attribute("type",
537 "The exception class.")
539 value = Attribute("value",
540 "The exception instance.")
542 lineno = Attribute("lineno",
543 "The line number the error occurred on in the source.")
545 offset = Attribute("offset",
546 "The character offset at which the error occurred.")
548 Common definitions used by TAL and METAL compilation an transformation.
551 from types import ListType, TupleType
556 XML_NS = "http://www.w3.org/XML/1998/namespace" # URI for XML namespace
557 XMLNS_NS = "http://www.w3.org/2000/xmlns/" # URI for XML NS declarations
559 ZOPE_TAL_NS = "http://xml.zope.org/namespaces/tal"
560 ZOPE_METAL_NS = "http://xml.zope.org/namespaces/metal"
561 ZOPE_I18N_NS = "http://xml.zope.org/namespaces/i18n"
563 NAME_RE = "[a-zA-Z_][-a-zA-Z0-9_]*"
565 KNOWN_METAL_ATTRIBUTES = [
573 KNOWN_TAL_ATTRIBUTES = [
585 KNOWN_I18N_ATTRIBUTES = [
595 class TALError(Exception):
597 def __init__(self, msg, position=(None, None)):
600 self.lineno = position[0]
601 self.offset = position[1]
604 def setFile(self, filename):
605 self.filename = filename
609 if self.lineno is not None:
610 result = result + ", at line %d" % self.lineno
611 if self.offset is not None:
612 result = result + ", column %d" % (self.offset + 1)
613 if self.filename is not None:
614 result = result + ', in file %s' % self.filename
617 class METALError(TALError):
620 class TALESError(TALError):
623 class I18NError(TALError):
629 __implements__ = ITALESErrorInfo
631 def __init__(self, err, position=(None, None)):
632 if isinstance(err, Exception):
633 self.type = err.__class__
638 self.lineno = position[0]
639 self.offset = position[1]
644 _attr_re = re.compile(r"\s*([^\s]+)\s+([^\s].*)\Z", re.S)
645 _subst_re = re.compile(r"\s*(?:(text|raw|structure)\s+)?(.*)\Z", re.S)
648 def parseAttributeReplacements(arg, xml):
650 for part in splitParts(arg):
651 m = _attr_re.match(part)
653 raise TALError("Bad syntax in attributes: " + `part`)
654 name, expr = m.group(1, 2)
657 if dict.has_key(name):
658 raise TALError("Duplicate attribute name in attributes: " + `part`)
662 def parseSubstitution(arg, position=(None, None)):
663 m = _subst_re.match(arg)
665 raise TALError("Bad syntax in substitution text: " + `arg`, position)
666 key, expr = m.group(1, 2)
672 arg = arg.replace(";;", "\0")
673 parts = arg.split(';')
674 parts = [p.replace("\0", ";") for p in parts]
675 if len(parts) > 1 and not parts[-1].strip():
676 del parts[-1] # It ended in a semicolon
679 def isCurrentVersion(program):
680 version = getProgramVersion(program)
681 return version == TAL_VERSION
683 def getProgramMode(program):
684 version = getProgramVersion(program)
685 if (version == TAL_VERSION and isinstance(program[1], TupleType) and
686 len(program[1]) == 2):
687 opcode, mode = program[1]
692 def getProgramVersion(program):
693 if (len(program) >= 2 and
694 isinstance(program[0], TupleType) and len(program[0]) == 2):
695 opcode, version = program[0]
696 if opcode == "version":
701 _ent1_re = re.compile('&(?![A-Z#])', re.I)
702 _entch_re = re.compile('&([A-Z][A-Z0-9]*)(?![A-Z0-9;])', re.I)
703 _entn1_re = re.compile('&#(?![0-9X])', re.I)
704 _entnx_re = re.compile('&(#X[A-F0-9]*)(?![A-F0-9;])', re.I)
705 _entnd_re = re.compile('&(#[0-9][0-9]*)(?![0-9;])')
709 """Replace special characters '&<>' by character entities,
710 except when '&' already begins a syntactically valid entity."""
711 s = _ent1_re.sub('&', s)
712 s = _entch_re.sub(r'&\1', s)
713 s = _entn1_re.sub('&#', s)
714 s = _entnx_re.sub(r'&\1', s)
715 s = _entnd_re.sub(r'&\1', s)
716 s = s.replace('<', '<')
717 s = s.replace('>', '>')
718 s = s.replace('"', '"')
721 Code generator for TALInterpreter intermediate code.
733 _name_rx = re.compile(NAME_RE)
742 def __init__(self, expressionCompiler=None, xml=1, source_file=None):
743 if not expressionCompiler:
744 expressionCompiler = AthanaTALEngine()
745 self.expressionCompiler = expressionCompiler
746 self.CompilerError = expressionCompiler.getCompilerError()
754 self.emit("version", TAL_VERSION)
755 self.emit("mode", xml and "xml" or "html")
756 if source_file is not None:
757 self.source_file = source_file
758 self.emit("setSourceFile", source_file)
759 self.i18nContext = TranslationContext()
763 assert not self.stack
764 assert not self.todoStack
765 return self.optimize(self.program), self.macros
767 def optimize(self, program):
775 for cursor in xrange(len(program)+1):
777 item = program[cursor]
781 if opcode == "rawtext":
782 collect.append(item[1])
784 if opcode == "endTag":
785 collect.append("</%s>" % item[1])
787 if opcode == "startTag":
788 if self.optimizeStartTag(collect, item[1], item[2], ">"):
790 if opcode == "startEndTag":
791 if self.optimizeStartTag(collect, item[1], item[2], endsep):
793 if opcode in ("beginScope", "endScope"):
794 output.append(self.optimizeArgsList(item))
799 text = "".join(collect)
803 i = len(text) - (i + 1)
804 output.append(("rawtextColumn", (text, i)))
806 output.append(("rawtextOffset", (text, len(text))))
808 output.append(self.optimizeArgsList(item))
810 return self.optimizeCommonTriple(output)
812 def optimizeArgsList(self, item):
816 return item[0], tuple(item[1:])
818 def optimizeStartTag(self, collect, name, attrlist, end):
820 collect.append("<%s%s" % (name, end))
824 for i in range(len(attrlist)):
828 name, value, action = item[:3]
829 attrlist[i] = (name, value, action) + item[3:]
834 s = '%s="%s"' % (item[0], attrEscape(item[1]))
835 attrlist[i] = item[0], s
842 def optimizeCommonTriple(self, program):
846 prev2, prev1 = output
847 for item in program[2:]:
848 if ( item[0] == "beginScope"
849 and prev1[0] == "setPosition"
850 and prev2[0] == "rawtextColumn"):
851 position = output.pop()[1]
852 text, column = output.pop()[1]
855 if output and output[-1][0] == "endScope":
858 item = ("rawtextBeginScope",
859 (text, column, position, closeprev, item[1]))
865 def todoPush(self, todo):
866 self.todoStack.append(todo)
869 return self.todoStack.pop()
871 def compileExpression(self, expr):
873 return self.expressionCompiler.compile(expr)
874 except self.CompilerError, err:
875 raise TALError('%s in expression %s' % (err.args[0], `expr`),
878 def pushProgram(self):
879 self.stack.append(self.program)
882 def popProgram(self):
883 program = self.program
884 self.program = self.stack.pop()
885 return self.optimize(program)
888 self.slotStack.append(self.slots)
893 self.slots = self.slotStack.pop()
896 def emit(self, *instruction):
897 self.program.append(instruction)
899 def emitStartTag(self, name, attrlist, isend=0):
901 opcode = "startEndTag"
904 self.emit(opcode, name, attrlist)
906 def emitEndTag(self, name):
907 if self.xml and self.program and self.program[-1][0] == "startTag":
908 self.program[-1] = ("startEndTag",) + self.program[-1][1:]
910 self.emit("endTag", name)
912 def emitOptTag(self, name, optTag, isend):
913 program = self.popProgram() #block
914 start = self.popProgram() #start tag
915 if (isend or not program) and self.xml:
916 start[-1] = ("startEndTag",) + start[-1][1:]
920 cexpr = self.compileExpression(optTag[0])
921 self.emit("optTag", name, cexpr, optTag[1], isend, start, program)
923 def emitRawText(self, text):
924 self.emit("rawtext", text)
926 def emitText(self, text):
927 self.emitRawText(cgi.escape(text))
929 def emitDefines(self, defines):
930 for part in splitParts(defines):
932 r"(?s)\s*(?:(global|local)\s+)?(%s)\s+(.*)\Z" % NAME_RE, part)
934 raise TALError("invalid define syntax: " + `part`,
936 scope, name, expr = m.group(1, 2, 3)
937 scope = scope or "local"
938 cexpr = self.compileExpression(expr)
940 self.emit("setLocal", name, cexpr)
942 self.emit("setGlobal", name, cexpr)
944 def emitOnError(self, name, onError, TALtag, isend):
945 block = self.popProgram()
946 key, expr = parseSubstitution(onError)
947 cexpr = self.compileExpression(expr)
949 self.emit("insertText", cexpr, [])
951 self.emit("insertRaw", cexpr, [])
953 assert key == "structure"
954 self.emit("insertStructure", cexpr, {}, [])
956 self.emitOptTag(name, (None, 1), isend)
958 self.emitEndTag(name)
959 handler = self.popProgram()
960 self.emit("onError", block, handler)
962 def emitCondition(self, expr):
963 cexpr = self.compileExpression(expr)
964 program = self.popProgram()
965 self.emit("condition", cexpr, program)
967 def emitRepeat(self, arg):
970 m = re.match("(?s)\s*(%s)\s+(.*)\Z" % NAME_RE, arg)
972 raise TALError("invalid repeat syntax: " + `arg`,
974 name, expr = m.group(1, 2)
975 cexpr = self.compileExpression(expr)
976 program = self.popProgram()
977 self.emit("loop", name, cexpr, program)
980 def emitSubstitution(self, arg, attrDict={}):
981 key, expr = parseSubstitution(arg)
982 cexpr = self.compileExpression(expr)
983 program = self.popProgram()
985 self.emit("insertText", cexpr, program)
987 self.emit("insertRaw", cexpr, program)
989 assert key == "structure"
990 self.emit("insertStructure", cexpr, attrDict, program)
992 def emitI18nVariable(self, stuff):
993 varname, action, expression = stuff
994 m = _name_rx.match(varname)
995 if m is None or m.group() != varname:
996 raise TALError("illegal i18n:name: %r" % varname, self.position)
998 program = self.popProgram()
999 if action == I18N_REPLACE:
1000 program = program[1:-1]
1001 elif action == I18N_CONTENT:
1004 assert action == I18N_EXPRESSION
1005 key, expr = parseSubstitution(expression)
1006 cexpr = self.compileExpression(expr)
1007 self.emit('i18nVariable',
1008 varname, program, cexpr, int(key == "structure"))
1010 def emitTranslation(self, msgid, i18ndata):
1011 program = self.popProgram()
1012 if i18ndata is None:
1013 self.emit('insertTranslation', msgid, program)
1015 key, expr = parseSubstitution(i18ndata)
1016 cexpr = self.compileExpression(expr)
1017 assert key == 'text'
1018 self.emit('insertTranslation', msgid, program, cexpr)
1020 def emitDefineMacro(self, macroName):
1021 program = self.popProgram()
1022 macroName = macroName.strip()
1023 if self.macros.has_key(macroName):
1024 raise METALError("duplicate macro definition: %s" % `macroName`,
1026 if not re.match('%s$' % NAME_RE, macroName):
1027 raise METALError("invalid macro name: %s" % `macroName`,
1029 self.macros[macroName] = program
1030 self.inMacroDef = self.inMacroDef - 1
1031 self.emit("defineMacro", macroName, program)
1033 def emitUseMacro(self, expr):
1034 cexpr = self.compileExpression(expr)
1035 program = self.popProgram()
1037 self.emit("useMacro", expr, cexpr, self.popSlots(), program)
1039 def emitDefineSlot(self, slotName):
1040 program = self.popProgram()
1041 slotName = slotName.strip()
1042 if not re.match('%s$' % NAME_RE, slotName):
1043 raise METALError("invalid slot name: %s" % `slotName`,
1045 self.emit("defineSlot", slotName, program)
1047 def emitFillSlot(self, slotName):
1048 program = self.popProgram()
1049 slotName = slotName.strip()
1050 if self.slots.has_key(slotName):
1051 raise METALError("duplicate fill-slot name: %s" % `slotName`,
1053 if not re.match('%s$' % NAME_RE, slotName):
1054 raise METALError("invalid slot name: %s" % `slotName`,
1056 self.slots[slotName] = program
1058 self.emit("fillSlot", slotName, program)
1060 def unEmitWhitespace(self):
1062 i = len(self.program) - 1
1064 item = self.program[i]
1065 if item[0] != "rawtext":
1068 if not re.match(r"\A\s*\Z", text):
1070 collect.append(text)
1072 del self.program[i+1:]
1073 if i >= 0 and self.program[i][0] == "rawtext":
1074 text = self.program[i][1]
1075 m = re.search(r"\s+\Z", text)
1077 self.program[i] = ("rawtext", text[:m.start()])
1078 collect.append(m.group())
1080 return "".join(collect)
1082 def unEmitNewlineWhitespace(self):
1084 i = len(self.program)
1087 item = self.program[i]
1088 if item[0] != "rawtext":
1091 if re.match(r"\A[ \t]*\Z", text):
1092 collect.append(text)
1094 m = re.match(r"(?s)^(.*)(\n[ \t]*)\Z", text)
1097 text, rest = m.group(1, 2)
1099 rest = rest + "".join(collect)
1100 del self.program[i:]
1102 self.emit("rawtext", text)
1106 def replaceAttrs(self, attrlist, repldict):
1110 for item in attrlist:
1112 if repldict.has_key(key):
1113 expr, xlat, msgid = repldict[key]
1114 item = item[:2] + ("replace", expr, xlat, msgid)
1116 newlist.append(item)
1117 for key, (expr, xlat, msgid) in repldict.items():
1118 newlist.append((key, None, "insert", expr, xlat, msgid))
1121 def emitStartElement(self, name, attrlist, taldict, metaldict, i18ndict,
1122 position=(None, None), isend=0):
1123 if not taldict and not metaldict and not i18ndict:
1124 self.emitStartTag(name, attrlist, isend)
1127 self.emitEndElement(name, isend)
1130 self.position = position
1131 for key, value in taldict.items():
1132 if key not in KNOWN_TAL_ATTRIBUTES:
1133 raise TALError("bad TAL attribute: " + `key`, position)
1134 if not (value or key == 'omit-tag'):
1135 raise TALError("missing value for TAL attribute: " +
1137 for key, value in metaldict.items():
1138 if key not in KNOWN_METAL_ATTRIBUTES:
1139 raise METALError("bad METAL attribute: " + `key`,
1142 raise TALError("missing value for METAL attribute: " +
1144 for key, value in i18ndict.items():
1145 if key not in KNOWN_I18N_ATTRIBUTES:
1146 raise I18NError("bad i18n attribute: " + `key`, position)
1147 if not value and key in ("attributes", "data", "id"):
1148 raise I18NError("missing value for i18n attribute: " +
1151 defineMacro = metaldict.get("define-macro")
1152 useMacro = metaldict.get("use-macro")
1153 defineSlot = metaldict.get("define-slot")
1154 fillSlot = metaldict.get("fill-slot")
1155 define = taldict.get("define")
1156 condition = taldict.get("condition")
1157 repeat = taldict.get("repeat")
1158 content = taldict.get("content")
1159 replace = taldict.get("replace")
1160 attrsubst = taldict.get("attributes")
1161 onError = taldict.get("on-error")
1162 omitTag = taldict.get("omit-tag")
1163 TALtag = taldict.get("tal tag")
1164 i18nattrs = i18ndict.get("attributes")
1165 msgid = i18ndict.get("translate")
1166 varname = i18ndict.get('name')
1167 i18ndata = i18ndict.get('data')
1169 if varname and not self.i18nLevel:
1171 "i18n:name can only occur inside a translation unit",
1174 if i18ndata and not msgid:
1175 raise I18NError("i18n:data must be accompanied by i18n:translate",
1178 if len(metaldict) > 1 and (defineMacro or useMacro):
1179 raise METALError("define-macro and use-macro cannot be used "
1180 "together or with define-slot or fill-slot",
1185 "tal:content and tal:replace are mutually exclusive",
1187 if msgid is not None:
1189 "i18n:translate and tal:replace are mutually exclusive",
1192 repeatWhitespace = None
1194 repeatWhitespace = self.unEmitNewlineWhitespace()
1195 if position != (None, None):
1196 self.emit("setPosition", position)
1200 if self.source_file is not None:
1201 self.emit("setSourceFile", self.source_file)
1202 todo["fillSlot"] = fillSlot
1206 raise METALError("fill-slot must be within a use-macro",
1208 if not self.inMacroUse:
1211 self.emit("version", TAL_VERSION)
1212 self.emit("mode", self.xml and "xml" or "html")
1213 if self.source_file is not None:
1214 self.emit("setSourceFile", self.source_file)
1215 todo["defineMacro"] = defineMacro
1216 self.inMacroDef = self.inMacroDef + 1
1220 todo["useMacro"] = useMacro
1223 if not self.inMacroDef:
1225 "define-slot must be within a define-macro",
1228 todo["defineSlot"] = defineSlot
1230 if defineSlot or i18ndict:
1232 domain = i18ndict.get("domain") or self.i18nContext.domain
1233 source = i18ndict.get("source") or self.i18nContext.source
1234 target = i18ndict.get("target") or self.i18nContext.target
1235 if ( domain != DEFAULT_DOMAIN
1236 or source is not None
1237 or target is not None):
1238 self.i18nContext = TranslationContext(self.i18nContext,
1242 self.emit("beginI18nContext",
1243 {"domain": domain, "source": source,
1245 todo["i18ncontext"] = 1
1246 if taldict or i18ndict:
1248 for item in attrlist:
1249 key, value = item[:2]
1251 self.emit("beginScope", dict)
1254 self.pushProgram() # handler
1256 self.pushProgram() # start
1257 self.emitStartTag(name, list(attrlist)) # Must copy attrlist!
1259 self.pushProgram() # start
1260 self.pushProgram() # block
1261 todo["onError"] = onError
1263 self.emitDefines(define)
1264 todo["define"] = define
1267 todo["condition"] = condition
1269 todo["repeat"] = repeat
1271 if repeatWhitespace:
1272 self.emitText(repeatWhitespace)
1275 todo['i18nvar'] = (varname, I18N_CONTENT, None)
1276 todo["content"] = content
1279 todo["content"] = content
1282 todo['i18nvar'] = (varname, I18N_EXPRESSION, replace)
1284 todo["replace"] = replace
1287 todo['i18nvar'] = (varname, I18N_REPLACE, None)
1289 if msgid is not None:
1291 todo['msgid'] = msgid
1293 todo['i18ndata'] = i18ndata
1294 optTag = omitTag is not None or TALtag
1296 todo["optional tag"] = omitTag, TALtag
1298 if attrsubst or i18nattrs:
1300 repldict = parseAttributeReplacements(attrsubst,
1305 i18nattrs = _parseI18nAttributes(i18nattrs, attrlist, repldict,
1306 self.position, self.xml,
1310 for key, value in repldict.items():
1311 if i18nattrs.get(key, None):
1313 ("attribute [%s] cannot both be part of tal:attributes" +
1314 " and have a msgid in i18n:attributes") % key,
1316 ce = self.compileExpression(value)
1317 repldict[key] = ce, key in i18nattrs, i18nattrs.get(key)
1318 for key in i18nattrs:
1319 if not repldict.has_key(key):
1320 repldict[key] = None, 1, i18nattrs.get(key)
1324 todo["repldict"] = repldict
1326 self.emitStartTag(name, self.replaceAttrs(attrlist, repldict), isend)
1329 if content and not varname:
1331 if msgid is not None:
1333 if content and varname:
1335 if todo and position != (None, None):
1336 todo["position"] = position
1339 self.emitEndElement(name, isend)
1341 def emitEndElement(self, name, isend=0, implied=0):
1342 todo = self.todoPop()
1345 self.emitEndTag(name)
1348 self.position = position = todo.get("position", (None, None))
1349 defineMacro = todo.get("defineMacro")
1350 useMacro = todo.get("useMacro")
1351 defineSlot = todo.get("defineSlot")
1352 fillSlot = todo.get("fillSlot")
1353 repeat = todo.get("repeat")
1354 content = todo.get("content")
1355 replace = todo.get("replace")
1356 condition = todo.get("condition")
1357 onError = todo.get("onError")
1358 repldict = todo.get("repldict", {})
1359 scope = todo.get("scope")
1360 optTag = todo.get("optional tag")
1361 msgid = todo.get('msgid')
1362 i18ncontext = todo.get("i18ncontext")
1363 varname = todo.get('i18nvar')
1364 i18ndata = todo.get('i18ndata')
1367 if defineMacro or useMacro or defineSlot or fillSlot:
1373 raise exc("%s attributes on <%s> require explicit </%s>" %
1374 (what, name, name), position)
1377 self.emitSubstitution(content, {})
1378 if msgid is not None:
1379 if (not varname) or (
1380 varname and (varname[1] == I18N_CONTENT)):
1381 self.emitTranslation(msgid, i18ndata)
1384 self.emitOptTag(name, optTag, isend)
1388 self.emitEndTag(name)
1390 self.emitSubstitution(replace, repldict)
1393 in [I18N_REPLACE, I18N_CONTENT, I18N_EXPRESSION])
1394 self.emitI18nVariable(varname)
1395 if msgid is not None:
1396 if varname and (varname[1] <> I18N_CONTENT):
1397 self.emitTranslation(msgid, i18ndata)
1399 self.emitRepeat(repeat)
1401 self.emitCondition(condition)
1403 self.emitOnError(name, onError, optTag and optTag[1], isend)
1405 self.emit("endScope")
1407 self.emit("endI18nContext")
1408 assert self.i18nContext.parent is not None
1409 self.i18nContext = self.i18nContext.parent
1411 self.emitDefineSlot(defineSlot)
1413 self.emitFillSlot(fillSlot)
1415 self.emitUseMacro(useMacro)
1417 self.emitDefineMacro(defineMacro)
1419 def _parseI18nAttributes(i18nattrs, attrlist, repldict, position,
1422 def addAttribute(dic, attr, msgid, position, xml):
1427 "attribute may only be specified once in i18n:attributes: "
1433 if ';' in i18nattrs:
1434 i18nattrlist = i18nattrs.split(';')
1435 i18nattrlist = [attr.strip().split()
1436 for attr in i18nattrlist if attr.strip()]
1437 for parts in i18nattrlist:
1439 raise TALError("illegal i18n:attributes specification: %r"
1446 addAttribute(d, attr, msgid, position, xml)
1448 i18nattrlist = i18nattrs.split()
1449 if len(i18nattrlist) == 1:
1450 addAttribute(d, i18nattrlist[0], None, position, xml)
1451 elif len(i18nattrlist) == 2:
1452 staticattrs = [attr[0] for attr in attrlist if len(attr) == 2]
1453 if (not i18nattrlist[1] in staticattrs) and (
1454 not i18nattrlist[1] in repldict):
1455 attr, msgid = i18nattrlist
1456 addAttribute(d, attr, msgid, position, xml)
1459 warnings.warn(I18N_ATTRIBUTES_WARNING
1460 % (source_file, str(position), i18nattrs)
1461 , DeprecationWarning)
1463 for attr in i18nattrlist:
1464 addAttribute(d, attr, msgid, position, xml)
1467 warnings.warn(I18N_ATTRIBUTES_WARNING
1468 % (source_file, str(position), i18nattrs)
1469 , DeprecationWarning)
1471 for attr in i18nattrlist:
1472 addAttribute(d, attr, msgid, position, xml)
1475 I18N_ATTRIBUTES_WARNING = (
1476 'Space separated attributes in i18n:attributes'
1477 ' are deprecated (i18n:attributes="value title"). Please use'
1478 ' semicolon to separate attributes'
1479 ' (i18n:attributes="value; title").'
1480 '\nFile %s at row, column %s\nAttributes %s')
1482 """Interpreter for a pre-compiled TAL program.
1489 from cgi import escape
1491 from StringIO import StringIO
1495 class ConflictError:
1503 BOOLEAN_HTML_ATTRS = [
1504 "compact", "nowrap", "ismap", "declare", "noshade", "checked",
1505 "disabled", "readonly", "multiple", "selected", "noresize",
1511 for s in BOOLEAN_HTML_ATTRS:
1515 BOOLEAN_HTML_ATTRS = _init()
1518 _spacejoin = ' '.join
1520 def normalize(text):
1521 return _spacejoin(text.split())
1524 NAME_RE = r"[a-zA-Z][a-zA-Z0-9_]*"
1525 _interp_regex = re.compile(r'(?<!\$)(\$(?:%(n)s|{%(n)s}))' %({'n': NAME_RE}))
1526 _get_var_regex = re.compile(r'%(n)s' %({'n': NAME_RE}))
1528 def interpolate(text, mapping):
1529 """Interpolate ${keyword} substitutions.
1531 This is called when no translation is provided by the translation
1536 to_replace = _interp_regex.findall(text)
1537 for string in to_replace:
1538 var = _get_var_regex.findall(string)[0]
1539 if mapping.has_key(var):
1540 subst = ustr(mapping[var])
1542 text = text.replace(string, subst)
1543 except UnicodeError:
1544 subst = `subst`[1:-1]
1545 text = text.replace(string, subst)
1549 class AltTALGenerator(TALGenerator):
1551 def __init__(self, repldict, expressionCompiler=None, xml=0):
1552 self.repldict = repldict
1554 TALGenerator.__init__(self, expressionCompiler, xml)
1556 def enable(self, enabled):
1557 self.enabled = enabled
1559 def emit(self, *args):
1561 TALGenerator.emit(self, *args)
1563 def emitStartElement(self, name, attrlist, taldict, metaldict, i18ndict,
1564 position=(None, None), isend=0):
1568 if self.enabled and self.repldict:
1569 taldict["attributes"] = "x x"
1570 TALGenerator.emitStartElement(self, name, attrlist,
1571 taldict, metaldict, i18ndict,
1574 def replaceAttrs(self, attrlist, repldict):
1575 if self.enabled and self.repldict:
1576 repldict = self.repldict
1577 self.repldict = None
1578 return TALGenerator.replaceAttrs(self, attrlist, repldict)
1581 class TALInterpreter:
1583 def __init__(self, program, macros, engine, stream=None,
1584 debug=0, wrap=60, metal=1, tal=1, showtal=-1,
1585 strictinsert=1, stackLimit=100, i18nInterpolate=1):
1586 self.program = program
1587 self.macros = macros
1588 self.engine = engine # Execution engine (aka context)
1589 self.Default = engine.getDefault()
1590 self.stream = stream or sys.stdout
1591 self._stream_write = self.stream.write
1597 self.dispatch = self.bytecode_handlers_tal
1599 self.dispatch = self.bytecode_handlers
1600 assert showtal in (-1, 0, 1)
1603 self.showtal = showtal
1604 self.strictinsert = strictinsert
1605 self.stackLimit = stackLimit
1608 self.endlen = len(self.endsep)
1609 self.macroStack = []
1610 self.position = None, None # (lineno, offset)
1614 self.sourceFile = None
1616 self.i18nInterpolate = i18nInterpolate
1617 self.i18nContext = TranslationContext()
1620 return FasterStringIO()
1622 def saveState(self):
1623 return (self.position, self.col, self.stream,
1624 self.scopeLevel, self.level, self.i18nContext)
1626 def restoreState(self, state):
1627 (self.position, self.col, self.stream,
1628 scopeLevel, level, i18n) = state
1629 self._stream_write = self.stream.write
1630 assert self.level == level
1631 while self.scopeLevel > scopeLevel:
1632 self.engine.endScope()
1633 self.scopeLevel = self.scopeLevel - 1
1634 self.engine.setPosition(self.position)
1635 self.i18nContext = i18n
1637 def restoreOutputState(self, state):
1638 (dummy, self.col, self.stream,
1639 scopeLevel, level, i18n) = state
1640 self._stream_write = self.stream.write
1641 assert self.level == level
1642 assert self.scopeLevel == scopeLevel
1644 def pushMacro(self, macroName, slots, entering=1):
1645 if len(self.macroStack) >= self.stackLimit:
1646 raise METALError("macro nesting limit (%d) exceeded "
1647 "by %s" % (self.stackLimit, `macroName`))
1648 self.macroStack.append([macroName, slots, entering, self.i18nContext])
1651 return self.macroStack.pop()
1654 assert self.level == 0
1655 assert self.scopeLevel == 0
1656 assert self.i18nContext.parent is None
1657 self.interpret(self.program)
1658 assert self.level == 0
1659 assert self.scopeLevel == 0
1660 assert self.i18nContext.parent is None
1662 self._stream_write("\n")
1665 def interpretWithStream(self, program, stream):
1666 oldstream = self.stream
1667 self.stream = stream
1668 self._stream_write = stream.write
1670 self.interpret(program)
1672 self.stream = oldstream
1673 self._stream_write = oldstream.write
1675 def stream_write(self, s,
1677 self._stream_write(s)
1680 self.col = self.col + len(s)
1682 self.col = len(s) - (i + 1)
1684 bytecode_handlers = {}
1686 def interpret(self, program):
1687 oldlevel = self.level
1688 self.level = oldlevel + 1
1689 handlers = self.dispatch
1692 for (opcode, args) in program:
1693 s = "%sdo_%s(%s)\n" % (" "*self.level, opcode,
1696 s = s[:76] + "...\n"
1698 handlers[opcode](self, args)
1700 for (opcode, args) in program:
1701 handlers[opcode](self, args)
1703 self.level = oldlevel
1705 def do_version(self, version):
1706 assert version == TAL_VERSION
1707 bytecode_handlers["version"] = do_version
1709 def do_mode(self, mode):
1710 assert mode in ("html", "xml")
1711 self.html = (mode == "html")
1716 self.endlen = len(self.endsep)
1717 bytecode_handlers["mode"] = do_mode
1719 def do_setSourceFile(self, source_file):
1720 self.sourceFile = source_file
1721 self.engine.setSourceFile(source_file)
1722 bytecode_handlers["setSourceFile"] = do_setSourceFile
1724 def do_setPosition(self, position):
1725 self.position = position
1726 self.engine.setPosition(position)
1727 bytecode_handlers["setPosition"] = do_setPosition
1729 def do_startEndTag(self, stuff):
1730 self.do_startTag(stuff, self.endsep, self.endlen)
1731 bytecode_handlers["startEndTag"] = do_startEndTag
1733 def do_startTag(self, (name, attrList),
1734 end=">", endlen=1, _len=len):
1735 self._currentTag = name
1738 col = self.col + _len(name) + 1
1742 align = 4 # Avoid a narrow column far to the right
1743 attrAction = self.dispatch["<attrAction>"]
1745 for item in attrList:
1749 if item[2] in ('metal', 'tal', 'xmlns', 'i18n'):
1750 if not self.showtal:
1752 ok, name, s = self.attrAction(item)
1754 ok, name, s = attrAction(self, item)
1760 col + 1 + slen > wrap):
1766 col = col + 1 + slen
1771 self._stream_write(_nulljoin(L))
1773 bytecode_handlers["startTag"] = do_startTag
1775 def attrAction(self, item):
1776 name, value, action = item[:3]
1777 if action == 'insert':
1778 return 0, name, value
1779 macs = self.macroStack
1780 if action == 'metal' and self.metal and macs:
1781 if len(macs) > 1 or not macs[-1][2]:
1782 return 0, name, value
1784 i = name.rfind(":") + 1
1785 prefix, suffix = name[:i], name[i:]
1786 if suffix == "define-macro":
1787 name = prefix + "use-macro"
1788 value = macs[-1][0] # Macro name
1789 elif suffix == "define-slot":
1790 name = prefix + "fill-slot"
1791 elif suffix == "fill-slot":
1794 return 0, name, value
1799 value = '%s="%s"' % (name, attrEscape(value))
1800 return 1, name, value
1802 def attrAction_tal(self, item):
1803 name, value, action = item[:3]
1805 expr, xlat, msgid = item[3:]
1806 if self.html and name.lower() in BOOLEAN_HTML_ATTRS:
1807 evalue = self.engine.evaluateBoolean(item[3])
1808 if evalue is self.Default:
1809 if action == 'insert': # Cancelled insert
1815 elif expr is not None:
1816 evalue = self.engine.evaluateText(item[3])
1817 if evalue is self.Default:
1818 if action == 'insert': # Cancelled insert
1829 translated = self.translate(msgid or value, value, {})
1830 if translated is not None:
1834 elif evalue is self.Default:
1835 value = attrEscape(value)
1837 value = escape(value, quote=1)
1838 value = '%s="%s"' % (name, value)
1839 return ok, name, value
1840 bytecode_handlers["<attrAction>"] = attrAction
1842 def no_tag(self, start, program):
1843 state = self.saveState()
1844 self.stream = stream = self.StringIO()
1845 self._stream_write = stream.write
1846 self.interpret(start)
1847 self.restoreOutputState(state)
1848 self.interpret(program)
1850 def do_optTag(self, (name, cexpr, tag_ns, isend, start, program),
1852 if tag_ns and not self.showtal:
1853 return self.no_tag(start, program)
1855 self.interpret(start)
1857 self.interpret(program)
1859 self._stream_write(s)
1860 self.col = self.col + len(s)
1862 def do_optTag_tal(self, stuff):
1864 if cexpr is not None and (cexpr == '' or
1865 self.engine.evaluateBoolean(cexpr)):
1866 self.no_tag(stuff[-2], stuff[-1])
1868 self.do_optTag(stuff)
1869 bytecode_handlers["optTag"] = do_optTag
1871 def do_rawtextBeginScope(self, (s, col, position, closeprev, dict)):
1872 self._stream_write(s)
1874 self.position = position
1875 self.engine.setPosition(position)
1877 engine = self.engine
1881 self.engine.beginScope()
1882 self.scopeLevel = self.scopeLevel + 1
1884 def do_rawtextBeginScope_tal(self, (s, col, position, closeprev, dict)):
1885 self._stream_write(s)
1887 engine = self.engine
1888 self.position = position
1889 engine.setPosition(position)
1895 self.scopeLevel = self.scopeLevel + 1
1896 engine.setLocal("attrs", dict)
1897 bytecode_handlers["rawtextBeginScope"] = do_rawtextBeginScope
1899 def do_beginScope(self, dict):
1900 self.engine.beginScope()
1901 self.scopeLevel = self.scopeLevel + 1
1903 def do_beginScope_tal(self, dict):
1904 engine = self.engine
1906 engine.setLocal("attrs", dict)
1907 self.scopeLevel = self.scopeLevel + 1
1908 bytecode_handlers["beginScope"] = do_beginScope
1910 def do_endScope(self, notused=None):
1911 self.engine.endScope()
1912 self.scopeLevel = self.scopeLevel - 1
1913 bytecode_handlers["endScope"] = do_endScope
1915 def do_setLocal(self, notused):
1918 def do_setLocal_tal(self, (name, expr)):
1919 self.engine.setLocal(name, self.engine.evaluateValue(expr))
1920 bytecode_handlers["setLocal"] = do_setLocal
1922 def do_setGlobal_tal(self, (name, expr)):
1923 self.engine.setGlobal(name, self.engine.evaluateValue(expr))
1924 bytecode_handlers["setGlobal"] = do_setLocal
1926 def do_beginI18nContext(self, settings):
1928 self.i18nContext = TranslationContext(self.i18nContext,
1929 domain=get("domain"),
1930 source=get("source"),
1931 target=get("target"))
1932 bytecode_handlers["beginI18nContext"] = do_beginI18nContext
1934 def do_endI18nContext(self, notused=None):
1935 self.i18nContext = self.i18nContext.parent
1936 assert self.i18nContext is not None
1937 bytecode_handlers["endI18nContext"] = do_endI18nContext
1939 def do_insertText(self, stuff):
1940 self.interpret(stuff[1])
1942 def do_insertText_tal(self, stuff):
1943 text = self.engine.evaluateText(stuff[0])
1946 if text is self.Default:
1947 self.interpret(stuff[1])
1949 if isinstance(text, MessageID):
1950 text = self.engine.translate(text.domain, text, text.mapping)
1952 self._stream_write(s)
1955 self.col = self.col + len(s)
1957 self.col = len(s) - (i + 1)
1958 bytecode_handlers["insertText"] = do_insertText
1960 def do_insertRawText_tal(self, stuff):
1961 text = self.engine.evaluateText(stuff[0])
1964 if text is self.Default:
1965 self.interpret(stuff[1])
1967 if isinstance(text, MessageID):
1968 text = self.engine.translate(text.domain, text, text.mapping)
1970 self._stream_write(s)
1973 self.col = self.col + len(s)
1975 self.col = len(s) - (i + 1)
1977 def do_i18nVariable(self, stuff):
1978 varname, program, expression, structure = stuff
1979 if expression is None:
1980 state = self.saveState()
1982 tmpstream = self.StringIO()
1983 self.interpretWithStream(program, tmpstream)
1984 if self.html and self._currentTag == "pre":
1985 value = tmpstream.getvalue()
1987 value = normalize(tmpstream.getvalue())
1989 self.restoreState(state)
1992 value = self.engine.evaluateStructure(expression)
1994 value = self.engine.evaluate(expression)
1996 if isinstance(value, MessageID):
1997 value = self.engine.translate(value.domain, value,
2001 value = cgi.escape(ustr(value))
2003 i18ndict, srepr = self.i18nStack[-1]
2004 i18ndict[varname] = value
2005 placeholder = '${%s}' % varname
2006 srepr.append(placeholder)
2007 self._stream_write(placeholder)
2008 bytecode_handlers['i18nVariable'] = do_i18nVariable
2010 def do_insertTranslation(self, stuff):
2014 self.i18nStack.append((i18ndict, srepr))
2016 currentTag = self._currentTag
2017 tmpstream = self.StringIO()
2018 self.interpretWithStream(stuff[1], tmpstream)
2019 default = tmpstream.getvalue()
2021 if self.html and currentTag == "pre":
2024 msgid = normalize(default)
2025 self.i18nStack.pop()
2027 obj = self.engine.evaluate(stuff[2])
2028 xlated_msgid = self.translate(msgid, default, i18ndict, obj)
2029 assert xlated_msgid is not None
2030 self._stream_write(xlated_msgid)
2031 bytecode_handlers['insertTranslation'] = do_insertTranslation
2033 def do_insertStructure(self, stuff):
2034 self.interpret(stuff[2])
2036 def do_insertStructure_tal(self, (expr, repldict, block)):
2037 structure = self.engine.evaluateStructure(expr)
2038 if structure is None:
2040 if structure is self.Default:
2041 self.interpret(block)
2043 text = ustr(structure)
2044 if not (repldict or self.strictinsert):
2045 self.stream_write(text)
2048 self.insertHTMLStructure(text, repldict)
2050 self.insertXMLStructure(text, repldict)
2051 bytecode_handlers["insertStructure"] = do_insertStructure
2053 def insertHTMLStructure(self, text, repldict):
2054 gen = AltTALGenerator(repldict, self.engine.getCompiler(), 0)
2055 p = HTMLTALParser(gen) # Raises an exception if text is invalid
2057 program, macros = p.getCode()
2058 self.interpret(program)
2060 def insertXMLStructure(self, text, repldict):
2061 gen = AltTALGenerator(repldict, self.engine.getCompiler(), 0)
2064 p.parseFragment('<!DOCTYPE foo PUBLIC "foo" "bar"><foo>')
2066 p.parseFragment(text) # Raises an exception if text is invalid
2068 p.parseFragment('</foo>', 1)
2069 program, macros = gen.getCode()
2070 self.interpret(program)
2072 def do_loop(self, (name, expr, block)):
2073 self.interpret(block)
2075 def do_loop_tal(self, (name, expr, block)):
2076 iterator = self.engine.setRepeat(name, expr)
2077 while iterator.next():
2078 self.interpret(block)
2079 bytecode_handlers["loop"] = do_loop
2081 def translate(self, msgid, default, i18ndict, obj=None):
2083 i18ndict.update(obj)
2084 if not self.i18nInterpolate:
2086 return self.engine.translate(self.i18nContext.domain,
2087 msgid, i18ndict, default=default)
2089 def do_rawtextColumn(self, (s, col)):
2090 self._stream_write(s)
2092 bytecode_handlers["rawtextColumn"] = do_rawtextColumn
2094 def do_rawtextOffset(self, (s, offset)):
2095 self._stream_write(s)
2096 self.col = self.col + offset
2097 bytecode_handlers["rawtextOffset"] = do_rawtextOffset
2099 def do_condition(self, (condition, block)):
2100 if not self.tal or self.engine.evaluateBoolean(condition):
2101 self.interpret(block)
2102 bytecode_handlers["condition"] = do_condition
2104 def do_defineMacro(self, (macroName, macro)):
2105 macs = self.macroStack
2107 entering = macs[-1][2]
2110 self.interpret(macro)
2111 assert macs[-1] is None
2114 self.interpret(macro)
2115 bytecode_handlers["defineMacro"] = do_defineMacro
2117 def do_useMacro(self, (macroName, macroExpr, compiledSlots, block)):
2119 self.interpret(block)
2121 macro = self.engine.evaluateMacro(macroExpr)
2122 if macro is self.Default:
2125 if not isCurrentVersion(macro):
2126 raise METALError("macro %s has incompatible version %s" %
2127 (`macroName`, `getProgramVersion(macro)`),
2129 mode = getProgramMode(macro)
2130 #if mode != (self.html and "html" or "xml"):
2131 # raise METALError("macro %s has incompatible mode %s" %
2132 # (`macroName`, `mode`), self.position)
2134 self.pushMacro(macroName, compiledSlots)
2135 prev_source = self.sourceFile
2136 self.interpret(macro)
2137 if self.sourceFile != prev_source:
2138 self.engine.setSourceFile(prev_source)
2139 self.sourceFile = prev_source
2141 bytecode_handlers["useMacro"] = do_useMacro
2143 def do_fillSlot(self, (slotName, block)):
2144 self.interpret(block)
2145 bytecode_handlers["fillSlot"] = do_fillSlot
2147 def do_defineSlot(self, (slotName, block)):
2149 self.interpret(block)
2151 macs = self.macroStack
2152 if macs and macs[-1] is not None:
2153 macroName, slots = self.popMacro()[:2]
2154 slot = slots.get(slotName)
2155 if slot is not None:
2156 prev_source = self.sourceFile
2157 self.interpret(slot)
2158 if self.sourceFile != prev_source:
2159 self.engine.setSourceFile(prev_source)
2160 self.sourceFile = prev_source
2161 self.pushMacro(macroName, slots, entering=0)
2163 self.pushMacro(macroName, slots)
2164 self.interpret(block)
2165 bytecode_handlers["defineSlot"] = do_defineSlot
2167 def do_onError(self, (block, handler)):
2168 self.interpret(block)
2170 def do_onError_tal(self, (block, handler)):
2171 state = self.saveState()
2172 self.stream = stream = self.StringIO()
2173 self._stream_write = stream.write
2175 self.interpret(block)
2176 except ConflictError:
2179 exc = sys.exc_info()[1]
2180 self.restoreState(state)
2181 engine = self.engine
2183 error = engine.createErrorInfo(exc, self.position)
2184 engine.setLocal('error', error)
2186 self.interpret(handler)
2190 self.restoreOutputState(state)
2191 self.stream_write(stream.getvalue())
2192 bytecode_handlers["onError"] = do_onError
2194 bytecode_handlers_tal = bytecode_handlers.copy()
2195 bytecode_handlers_tal["rawtextBeginScope"] = do_rawtextBeginScope_tal
2196 bytecode_handlers_tal["beginScope"] = do_beginScope_tal
2197 bytecode_handlers_tal["setLocal"] = do_setLocal_tal
2198 bytecode_handlers_tal["setGlobal"] = do_setGlobal_tal
2199 bytecode_handlers_tal["insertStructure"] = do_insertStructure_tal
2200 bytecode_handlers_tal["insertText"] = do_insertText_tal
2201 bytecode_handlers_tal["insertRaw"] = do_insertRawText_tal
2202 bytecode_handlers_tal["loop"] = do_loop_tal
2203 bytecode_handlers_tal["onError"] = do_onError_tal
2204 bytecode_handlers_tal["<attrAction>"] = attrAction_tal
2205 bytecode_handlers_tal["optTag"] = do_optTag_tal
2208 class FasterStringIO(StringIO):
2209 """Append-only version of StringIO.
2211 This let's us have a much faster write() method.
2215 self.write = _write_ValueError
2216 StringIO.close(self)
2218 def seek(self, pos, mode=0):
2219 raise RuntimeError("FasterStringIO.seek() not allowed")
2222 self.buflist.append(s)
2223 self.len = self.pos = self.pos + len(s)
2226 def _write_ValueError(s):
2227 raise ValueError, "I/O operation on closed file"
2229 Parse XML and compile to TALInterpreter intermediate code.
2233 class TALParser(XMLParser):
2235 ordered_attributes = 1
2237 def __init__(self, gen=None): # Override
2238 XMLParser.__init__(self)
2240 gen = TALGenerator()
2243 self.nsDict = {XML_NS: 'xml'}
2247 return self.gen.getCode()
2249 def getWarnings(self):
2252 def StartNamespaceDeclHandler(self, prefix, uri):
2253 self.nsStack.append(self.nsDict.copy())
2254 self.nsDict[uri] = prefix
2255 self.nsNew.append((prefix, uri))
2257 def EndNamespaceDeclHandler(self, prefix):
2258 self.nsDict = self.nsStack.pop()
2260 def StartElementHandler(self, name, attrs):
2261 if self.ordered_attributes:
2263 for i in range(0, len(attrs), 2):
2266 attrlist.append((key, value))
2268 attrlist = attrs.items()
2269 attrlist.sort() # For definiteness
2270 name, attrlist, taldict, metaldict, i18ndict \
2271 = self.process_ns(name, attrlist)
2272 attrlist = self.xmlnsattrs() + attrlist
2273 self.gen.emitStartElement(name, attrlist, taldict, metaldict, i18ndict)
2275 def process_ns(self, name, attrlist):
2280 name, namebase, namens = self.fixname(name)
2281 for key, value in attrlist:
2282 key, keybase, keyns = self.fixname(key)
2283 ns = keyns or namens # default to tag namespace
2286 metaldict[keybase] = value
2287 item = item + ("metal",)
2289 taldict[keybase] = value
2290 item = item + ("tal",)
2292 i18ndict[keybase] = value
2293 item = item + ('i18n',)
2294 fixedattrlist.append(item)
2295 if namens in ('metal', 'tal', 'i18n'):
2296 taldict['tal tag'] = namens
2297 return name, fixedattrlist, taldict, metaldict, i18ndict
2299 def xmlnsattrs(self):
2301 for prefix, uri in self.nsNew:
2303 key = "xmlns:" + prefix
2306 if uri in (ZOPE_METAL_NS, ZOPE_TAL_NS, ZOPE_I18N_NS):
2307 item = (key, uri, "xmlns")
2310 newlist.append(item)
2314 def fixname(self, name):
2316 uri, name = name.split(' ')
2317 prefix = self.nsDict[uri]
2320 prefixed = "%s:%s" % (prefix, name)
2322 if uri == ZOPE_TAL_NS:
2324 elif uri == ZOPE_METAL_NS:
2326 elif uri == ZOPE_I18N_NS:
2328 return (prefixed, name, ns)
2329 return (name, name, None)
2331 def EndElementHandler(self, name):
2332 name = self.fixname(name)[0]
2333 self.gen.emitEndElement(name)
2335 def DefaultHandler(self, text):
2336 self.gen.emitRawText(text)
2338 """Translation context object for the TALInterpreter's I18N support.
2340 The translation context provides a container for the information
2341 needed to perform translation of a marked string from a page template.
2345 DEFAULT_DOMAIN = "default"
2347 class TranslationContext:
2348 """Information about the I18N settings of a TAL processor."""
2350 def __init__(self, parent=None, domain=None, target=None, source=None):
2353 domain = parent.domain
2355 target = parent.target
2357 source = parent.source
2358 elif domain is None:
2359 domain = DEFAULT_DOMAIN
2361 self.parent = parent
2362 self.domain = domain
2363 self.target = target
2364 self.source = source
2366 Dummy TALES engine so that I can test out the TAL implementation.
2377 Default = _Default()
2379 name_match = re.compile(r"(?s)(%s):(.*)\Z" % NAME_RE).match
2381 class CompilerError(Exception):
2384 class AthanaTALEngine:
2389 __implements__ = ITALESCompiler, ITALESEngine
2391 def __init__(self, macros=None, context=None, webcontext=None, language=None, request=None):
2394 self.macros = macros
2395 dict = {'nothing': None, 'default': Default}
2396 if context is not None:
2397 dict.update(context)
2399 self.locals = self.globals = dict
2401 self.webcontext = webcontext
2402 self.language = language
2403 self.request = request
2405 def compilefile(self, file, mode=None):
2406 assert mode in ("html", "xml", None)
2407 #file = join_paths(GLOBAL_ROOT_DIR,join_paths(self.webcontext.root, file))
2409 ext = os.path.splitext(file)[1]
2410 if ext.lower() in (".html", ".htm"):
2415 p = HTMLTALParser(TALGenerator(self))
2417 p = TALParser(TALGenerator(self))
2421 def getCompilerError(self):
2422 return CompilerError
2424 def getCompiler(self):
2427 def setSourceFile(self, source_file):
2428 self.source_file = source_file
2430 def setPosition(self, position):
2431 self.position = position
2433 def compile(self, expr):
2434 return "$%s$" % expr
2436 def uncompile(self, expression):
2437 assert (expression.startswith("$") and expression.endswith("$"),
2439 return expression[1:-1]
2441 def beginScope(self):
2442 self.stack.append(self.locals)
2445 assert len(self.stack) > 1, "more endScope() than beginScope() calls"
2446 self.locals = self.stack.pop()
2448 def setLocal(self, name, value):
2449 if self.locals is self.stack[-1]:
2450 self.locals = self.locals.copy()
2451 self.locals[name] = value
2453 def setGlobal(self, name, value):
2454 self.globals[name] = value
2456 def evaluate(self, expression):
2457 assert (expression.startswith("$") and expression.endswith("$"),
2459 expression = expression[1:-1]
2460 m = name_match(expression)
2462 type, expr = m.group(1, 2)
2466 if type in ("string", "str"):
2468 if type in ("path", "var", "global", "local"):
2469 return self.evaluatePathOrVar(expr)
2471 return not self.evaluate(expr)
2472 if type == "exists":
2473 return self.locals.has_key(expr) or self.globals.has_key(expr)
2474 if type == "python":
2476 return eval(expr, self.globals, self.locals)
2478 print "Error in python expression"
2479 print sys.exc_info()[0], sys.exc_info()[1]
2480 traceback.print_tb(sys.exc_info()[2])
2481 raise TALESError("evaluation error in %s" % `expr`)
2483 if type == "position":
2485 lineno, offset = self.position
2487 lineno, offset = None, None
2488 return '%s (%s,%s)' % (self.source_file, lineno, offset)
2489 raise TALESError("unrecognized expression: " + `expression`)
2491 def evaluatePathOrVar(self, expr):
2495 if expr.rfind("/")>0:
2499 if self.locals.has_key(_expr):
2501 return getattr(self.locals[_expr],_f)
2503 return self.locals[_expr]
2504 elif self.globals.has_key(_expr):
2506 return getattr(self.globals[_expr], _f)
2508 return self.globals[_expr]
2510 raise TALESError("unknown variable: %s" % `_expr`)
2512 def evaluateValue(self, expr):
2513 return self.evaluate(expr)
2515 def evaluateBoolean(self, expr):
2516 return self.evaluate(expr)
2518 def evaluateText(self, expr):
2519 text = self.evaluate(expr)
2520 if text is not None and text is not Default:
2524 def evaluateStructure(self, expr):
2525 return self.evaluate(expr)
2527 def evaluateSequence(self, expr):
2528 return self.evaluate(expr)
2530 def evaluateMacro(self, macroName):
2531 assert (macroName.startswith("$") and macroName.endswith("$"),
2533 macroName = macroName[1:-1]
2534 file, localName = self.findMacroFile(macroName)
2536 macro = self.macros[localName]
2538 program, macros = self.compilefile(file)
2539 macro = macros.get(localName)
2541 raise TALESError("macro %s not found in file %s" %
2545 def findMacroDocument(self, macroName):
2546 file, localName = self.findMacroFile(macroName)
2548 return file, localName
2549 doc = parsefile(file)
2550 return doc, localName
2552 def findMacroFile(self, macroName):
2554 raise TALESError("empty macro name")
2555 i = macroName.rfind('/')
2558 return None, macroName
2560 fileName = getMacroFile(macroName[:i])
2561 localName = macroName[i+1:]
2562 return fileName, localName
2564 def setRepeat(self, name, expr):
2565 seq = self.evaluateSequence(expr)
2566 self.locals[name] = Iterator(name, seq, self)
2567 return self.locals[name]
2569 def createErrorInfo(self, err, position):
2570 return ErrorInfo(err, position)
2572 def getDefault(self):
2575 def translate(self, domain, msgid, mapping, default=None):
2577 text = default or msgid
2578 for f in translators:
2579 text = f(msgid, language=self.language, request=self.request)
2581 text = f(msgid, language=self.language, request=self.request)
2582 if text and text!=msgid:
2586 def repl(m, mapping=mapping):
2587 return ustr(mapping[m.group(m.lastindex).lower()])
2588 return VARIABLE.sub(repl, text)
2593 def __init__(self, name, seq, engine):
2596 self.engine = engine
2600 self.index = i = self.nextIndex
2605 self.nextIndex = i+1
2606 self.engine.setLocal(self.name, item)
2611 return not self.index % 2
2615 return self.index % 2
2618 return self.nextIndex
2625 def first(self, name=None):
2626 if self.start: return 1
2627 return not self.same_part(name, self._last, self.item)
2629 def last(self, name=None):
2630 if self.end: return 1
2631 return not self.same_part(name, self.item, self._next)
2634 return len(self.seq)
2637 VARIABLE = re.compile(r'\$(?:(%s)|\{(%s)\})' % (NAME_RE, NAME_RE))
2642 def runTAL(writer, context=None, string=None, file=None, macro=None, language=None, request=None):
2645 file = getMacroFile(file)
2650 if string and not file:
2651 if string in parsed_strings:
2652 program,macros = parsed_strings[string]
2654 program,macros = None,None
2655 elif file and not string:
2656 if file in parsed_files:
2657 (program,macros,mtime) = parsed_files[file]
2658 mtime_file = os.stat(file)[stat.ST_MTIME]
2659 if mtime != mtime_file:
2660 program,macros = None,None
2663 program,macros,mtime = None,None,None
2665 if not (program and macros):
2666 if file and file.endswith("xml"):
2667 talparser = TALParser(TALGenerator(AthanaTALEngine()))
2669 talparser = HTMLTALParser(TALGenerator(AthanaTALEngine()))
2671 talparser.parseString(string)
2672 (program, macros) = talparser.getCode()
2673 parsed_strings[string] = (program,macros)
2675 talparser.parseFile(file)
2676 (program, macros) = talparser.getCode()
2677 parsed_files[file] = (program,macros,mtime)
2679 if macro and macro in macros:
2680 program = macros[macro]
2681 engine = AthanaTALEngine(macros, context, language=language, request=request)
2682 TALInterpreter(program, macros, engine, writer, wrap=0)()
2684 def processTAL(context=None, string=None, file=None, macro=None, language=None, request=None):
2688 def write(self,text):
2689 if type(text) == type(u''):
2690 self.string += text.encode("utf-8")
2696 runTAL(wr, context, string=string, file=file, macro=macro, language=language, request=request)
2697 return wr.getvalue()
2705 p = TALParser(TALGenerator(AthanaTALEngine()))
2710 program, macros = p.getCode()
2716 engine = AthanaTALEngine(macros, {'node': Node()})
2717 TALInterpreter(program, macros, engine, MyWriter(), wrap=0)()
2721 """Convert any object to a plain string or unicode string,
2722 minimising the chance of raising a UnicodeError. This
2723 even works with uncooperative objects like Exceptions
2725 if type(v) == type(""): #isinstance(v, basestring):
2728 fn = getattr(v,'__str__',None)
2731 if isinstance(v, basestring):
2734 raise ValueError('__str__ returned wrong type')
2738 # ================ MEDUSA ===============
2752 from cgi import escape
2753 from urllib import unquote, splitquery
2759 class async_chat (asyncore.dispatcher):
2760 """This is an abstract class. You must derive from this class, and add
2761 the two methods collect_incoming_data() and found_terminator()"""
2763 # these are overridable defaults
2765 ac_in_buffer_size = 4096
2766 ac_out_buffer_size = 4096
2768 def __init__ (self, conn=None):
2769 self.ac_in_buffer = ''
2770 self.ac_out_buffer = ''
2771 self.producer_fifo = fifo()
2772 asyncore.dispatcher.__init__ (self, conn)
2774 def collect_incoming_data(self, data):
2775 raise NotImplementedError, "must be implemented in subclass"
2777 def found_terminator(self):
2778 raise NotImplementedError, "must be implemented in subclass"
2780 def set_terminator (self, term):
2781 "Set the input delimiter. Can be a fixed string of any length, an integer, or None"
2782 self.terminator = term
2784 def get_terminator (self):
2785 return self.terminator
2787 # grab some more data from the socket,
2788 # throw it to the collector method,
2789 # check for the terminator,
2790 # if found, transition to the next state.
2792 def handle_read (self):
2795 data = self.recv (self.ac_in_buffer_size)
2796 except socket.error, why:
2800 self.ac_in_buffer = self.ac_in_buffer + data
2802 # Continue to search for self.terminator in self.ac_in_buffer,
2803 # while calling self.collect_incoming_data. The while loop
2804 # is necessary because we might read several data+terminator
2805 # combos with a single recv(1024).
2807 while self.ac_in_buffer:
2808 lb = len(self.ac_in_buffer)
2809 terminator = self.get_terminator()
2810 if terminator is None or terminator == '':
2811 # no terminator, collect it all
2812 self.collect_incoming_data (self.ac_in_buffer)
2813 self.ac_in_buffer = ''
2814 elif isinstance(terminator, int):
2815 # numeric terminator
2818 self.collect_incoming_data (self.ac_in_buffer)
2819 self.ac_in_buffer = ''
2820 self.terminator = self.terminator - lb
2822 self.collect_incoming_data (self.ac_in_buffer[:n])
2823 self.ac_in_buffer = self.ac_in_buffer[n:]
2825 self.found_terminator()
2828 # 1) end of buffer matches terminator exactly:
2829 # collect data, transition
2830 # 2) end of buffer matches some prefix:
2831 # collect data to the prefix
2832 # 3) end of buffer does not match any prefix:
2834 terminator_len = len(terminator)
2835 index = self.ac_in_buffer.find(terminator)
2837 # we found the terminator
2839 # don't bother reporting the empty string (source of subtle bugs)
2840 self.collect_incoming_data (self.ac_in_buffer[:index])
2841 self.ac_in_buffer = self.ac_in_buffer[index+terminator_len:]
2842 # This does the Right Thing if the terminator is changed here.
2843 self.found_terminator()
2845 # check for a prefix of the terminator
2846 index = find_prefix_at_end (self.ac_in_buffer, terminator)
2849 # we found a prefix, collect up to the prefix
2850 self.collect_incoming_data (self.ac_in_buffer[:-index])
2851 self.ac_in_buffer = self.ac_in_buffer[-index:]
2854 # no prefix, collect it all
2855 self.collect_incoming_data (self.ac_in_buffer)
2856 self.ac_in_buffer = ''
2858 def handle_write (self):
2859 self.initiate_send ()
2861 def handle_close (self):
2864 def push (self, data):
2865 self.producer_fifo.push (simple_producer (data))
2866 self.initiate_send()
2868 def push_with_producer (self, producer):
2869 self.producer_fifo.push (producer)
2870 self.initiate_send()
2872 def readable (self):
2873 "predicate for inclusion in the readable for select()"
2874 return (len(self.ac_in_buffer) <= self.ac_in_buffer_size)
2876 def writable (self):
2877 "predicate for inclusion in the writable for select()"
2878 # return len(self.ac_out_buffer) or len(self.producer_fifo) or (not self.connected)
2879 # this is about twice as fast, though not as clear.
2881 (self.ac_out_buffer == '') and
2882 self.producer_fifo.is_empty() and
2886 def close_when_done (self):
2887 "automatically close this channel once the outgoing queue is empty"
2888 self.producer_fifo.push (None)
2890 # refill the outgoing buffer by calling the more() method
2891 # of the first producer in the queue
2892 def refill_buffer (self):
2894 if len(self.producer_fifo):
2895 p = self.producer_fifo.first()
2896 # a 'None' in the producer fifo is a sentinel,
2897 # telling us to close the channel.
2899 if not self.ac_out_buffer:
2900 self.producer_fifo.pop()
2903 elif isinstance(p, str):
2904 self.producer_fifo.pop()
2905 self.ac_out_buffer = self.ac_out_buffer + p
2909 self.ac_out_buffer = self.ac_out_buffer + data
2912 self.producer_fifo.pop()
2916 def initiate_send (self):
2917 obs = self.ac_out_buffer_size
2918 # try to refill the buffer
2919 if (len (self.ac_out_buffer) < obs):
2920 self.refill_buffer()
2922 if self.ac_out_buffer and self.connected:
2923 # try to send the buffer
2925 num_sent = self.send (self.ac_out_buffer[:obs])
2927 self.ac_out_buffer = self.ac_out_buffer[num_sent:]
2929 except socket.error, why:
2933 def discard_buffers (self):
2935 self.ac_in_buffer = ''
2936 self.ac_out_buffer = ''
2937 while self.producer_fifo:
2938 self.producer_fifo.pop()
2941 class simple_producer:
2943 def __init__ (self, data, buffer_size=512):
2945 self.buffer_size = buffer_size
2948 if len (self.data) > self.buffer_size:
2949 result = self.data[:self.buffer_size]
2950 self.data = self.data[self.buffer_size:]
2958 def __init__ (self, list=None):
2965 return len(self.list)
2967 def is_empty (self):
2968 return self.list == []
2973 def push (self, data):
2974 self.list.append (data)
2978 return (1, self.list.pop(0))
2982 # Given 'haystack', see if any prefix of 'needle' is at its end. This
2983 # assumes an exact match has already been checked. Return the number of
2984 # characters matched.
2986 # f_p_a_e ("qwerty\r", "\r\n") => 1
2987 # f_p_a_e ("qwertydkjf", "\r\n") => 0
2988 # f_p_a_e ("qwerty\r\n", "\r\n") => <undefined>
2990 # this could maybe be made faster with a computed regex?
2991 # [answer: no; circa Python-2.0, Jan 2001]
2992 # new python: 28961/s
2993 # old python: 18307/s
2997 def find_prefix_at_end (haystack, needle):
2999 while l and not haystack.endswith(needle[:l]):
3004 "general-purpose counter"
3006 def __init__ (self, initial_value=0):
3007 self.value = initial_value
3009 def increment (self, delta=1):
3012 self.value = self.value + delta
3013 except OverflowError:
3014 self.value = long(self.value) + delta
3017 def decrement (self, delta=1):
3020 self.value = self.value - delta
3021 except OverflowError:
3022 self.value = long(self.value) - delta
3026 return long(self.value)
3028 def __nonzero__ (self):
3029 return self.value != 0
3031 def __repr__ (self):
3032 return '<counter value=%s at %x>' % (self.value, id(self))
3035 s = str(long(self.value))
3043 return ''.join (args)
3045 def join (seq, field=' '):
3046 return field.join (seq)
3049 return '(' + s + ')'
3051 short_days = ['sun','mon','tue','wed','thu','fri','sat']
3052 long_days = ['sunday','monday','tuesday','wednesday','thursday','friday','saturday']
3054 short_day_reg = group (join (short_days, '|'))
3055 long_day_reg = group (join (long_days, '|'))
3059 daymap[short_days[i]] = i
3060 daymap[long_days[i]] = i
3062 hms_reg = join (3 * [group('[0-9][0-9]')], ':')
3064 months = ['jan','feb','mar','apr','may','jun','jul','aug','sep','oct','nov','dec']
3068 monmap[months[i]] = i+1
3070 months_reg = group (join (months, '|'))
3072 # From draft-ietf-http-v11-spec-07.txt/3.3.1
3073 # Sun, 06 Nov 1994 08:49:37 GMT ; RFC 822, updated by RFC 1123
3074 # Sunday, 06-Nov-94 08:49:37 GMT ; RFC 850, obsoleted by RFC 1036
3075 # Sun Nov 6 08:49:37 1994 ; ANSI C's asctime() format
3078 rfc822_date = join (
3079 [concat (short_day_reg,','), # day
3080 group('[0-9][0-9]?'), # date
3082 group('[0-9]+'), # year
3083 hms_reg, # hour minute second
3089 rfc822_reg = re.compile (rfc822_date)
3091 def unpack_rfc822 (m):
3096 monmap[g(3)], # month
3107 rfc850_date = join (
3108 [concat (long_day_reg,','),
3110 [group ('[0-9][0-9]?'),
3122 rfc850_reg = re.compile (rfc850_date)
3123 # they actually unpack the same way
3124 def unpack_rfc850 (m):
3129 monmap[g(3)], # month
3139 # parsdate.parsedate - ~700/sec.
3140 # parse_http_date - ~1333/sec.
3142 def build_http_date (when):
3143 return time.strftime ('%a, %d %b %Y %H:%M:%S GMT', time.gmtime(when))
3147 def parse_http_date (d):
3149 d = string.lower (d)
3151 m = rfc850_reg.match (d)
3152 if m and m.end() == len(d):
3153 retval = int (time.mktime (unpack_rfc850(m)) - tz)
3155 m = rfc822_reg.match (d)
3156 if m and m.end() == len(d):
3158 retval = int (time.mktime (unpack_rfc822(m)) - tz)
3159 except OverflowError:
3163 # Thanks to Craig Silverstein <csilvers@google.com> for pointing
3164 # out the DST discrepancy
3165 if time.daylight and time.localtime(retval)[-1] == 1: # DST correction
3166 retval = retval + (tz - time.altzone)
3167 return retval - time_offset
3171 tmpfile = join_paths(GLOBAL_TEMP_DIR, "datetest"+str(random.random())+".tmp")
3172 open(tmpfile,"wb").close()
3173 time1 = os.stat(tmpfile)[stat.ST_MTIME]
3175 time2 = parse_http_date(build_http_date(time.time()))
3176 time_offset = time2-time1
3181 class simple_producer:
3182 "producer for a string"
3183 def __init__ (self, data, buffer_size=1024):
3185 self.buffer_size = buffer_size
3188 if len (self.data) > self.buffer_size:
3189 result = self.data[:self.buffer_size]
3190 self.data = self.data[self.buffer_size:]
3197 class file_producer:
3198 "producer wrapper for file[-like] objects"
3200 # match http_channel's outgoing buffer size
3201 out_buffer_size = 1<<16
3203 def __init__ (self, file):
3211 data = self.file.read (self.out_buffer_size)
3220 # A simple output producer. This one does not [yet] have
3221 # the safety feature builtin to the monitor channel: runaway
3222 # output will not be caught.
3224 # don't try to print from within any of the methods
3227 class output_producer:
3228 "Acts like an output file; suitable for capturing sys.stdout"
3229 def __init__ (self):
3232 def write (self, data):
3233 lines = string.splitfields (data, '\n')
3234 data = string.join (lines, '\r\n')
3235 self.data = self.data + data
3237 def writeline (self, line):
3238 self.data = self.data + line + '\r\n'
3240 def writelines (self, lines):
3241 self.data = self.data + string.joinfields (
3249 def softspace (self, *args):
3254 result = self.data[:512]
3255 self.data = self.data[512:]
3260 class composite_producer:
3261 "combine a fifo of producers into one"
3262 def __init__ (self, producers):
3263 self.producers = producers
3266 while len(self.producers):
3267 p = self.producers[0]
3272 self.producers.pop(0)
3277 class globbing_producer:
3279 'glob' the output from a producer into a particular buffer size.
3280 helps reduce the number of calls to send(). [this appears to
3281 gain about 30% performance on requests to a single channel]
3284 def __init__ (self, producer, buffer_size=1<<16):
3285 self.producer = producer
3287 self.buffer_size = buffer_size
3290 while len(self.buffer) < self.buffer_size:
3291 data = self.producer.more()
3293 self.buffer = self.buffer + data
3301 class hooked_producer:
3303 A producer that will call <function> when it empties,.
3304 with an argument of the number of bytes produced. Useful
3305 for logging/instrumentation purposes.
3308 def __init__ (self, producer, function):
3309 self.producer = producer
3310 self.function = function
3315 result = self.producer.more()
3317 self.producer = None
3318 self.function (self.bytes)
3320 self.bytes = self.bytes + len(result)
3325 # HTTP 1.1 emphasizes that an advertised Content-Length header MUST be
3326 # correct. In the face of Strange Files, it is conceivable that
3327 # reading a 'file' may produce an amount of data not matching that
3328 # reported by os.stat() [text/binary mode issues, perhaps the file is
3329 # being appended to, etc..] This makes the chunked encoding a True
3330 # Blessing, and it really ought to be used even with normal files.
3331 # How beautifully it blends with the concept of the producer.
3333 class chunked_producer:
3334 """A producer that implements the 'chunked' transfer coding for HTTP/1.1.
3335 Here is a sample usage:
3336 request['Transfer-Encoding'] = 'chunked'
3338 producers.chunked_producer (your_producer)
3343 def __init__ (self, producer, footers=None):
3344 self.producer = producer
3345 self.footers = footers
3349 data = self.producer.more()
3351 return '%x\r\n%s\r\n' % (len(data), data)
3353 self.producer = None
3355 return string.join (
3356 ['0'] + self.footers,
3364 class escaping_producer:
3366 "A producer that escapes a sequence of characters"
3367 " Common usage: escaping the CRLF.CRLF sequence in SMTP, NNTP, etc..."
3369 def __init__ (self, producer, esc_from='\r\n.', esc_to='\r\n..'):
3370 self.producer = producer
3371 self.esc_from = esc_from
3372 self.esc_to = esc_to
3374 self.find_prefix_at_end = find_prefix_at_end
3377 esc_from = self.esc_from
3378 esc_to = self.esc_to
3380 buffer = self.buffer + self.producer.more()
3383 buffer = string.replace (buffer, esc_from, esc_to)
3384 i = self.find_prefix_at_end (buffer, esc_from)
3387 self.buffer = buffer[-i:]
3390 # no prefix, return it all
3397 "Keep track of the last <size> log messages"
3398 def __init__ (self, logger, size=500):
3400 self.logger = logger
3403 def log (self, message):
3404 self.messages.append (strip_eol (message))
3405 if len (self.messages) > self.size:
3406 del self.messages[0]
3407 self.logger.log (message)
3410 def html_repr (object):
3411 so = escape (repr (object))
3412 if hasattr (object, 'hyper_respond'):
3413 return '<a href="/status/object/%d/">%s</a>' % (id (object), so)
3417 def html_reprs (list, front='', back=''):
3419 lambda x,f=front,b=back: '%s%s%s' % (f,x,b),
3420 map (lambda x: escape (html_repr(x)), list)
3425 # for example, tera, giga, mega, kilo
3426 # p_d (n, (1024, 1024, 1024, 1024))
3427 # smallest divider goes first - for example
3428 # minutes, hours, days
3429 # p_d (n, (60, 60, 24))
3431 def progressive_divide (n, parts):
3434 n, rem = divmod (n, part)
3440 def split_by_units (n, units, dividers, format_string):
3441 divs = progressive_divide (n, dividers)
3443 for i in range(len(units)):
3445 result.append (format_string % (divs[i], units[i]))
3448 return [format_string % (0, units[0])]
3452 def english_bytes (n):
3453 return split_by_units (
3455 ('','K','M','G','T'),
3456 (1024, 1024, 1024, 1024, 1024),
3460 def english_time (n):
3461 return split_by_units (
3463 ('secs', 'mins', 'hours', 'days', 'weeks', 'years'),
3464 ( 60, 60, 24, 7, 52),
3470 # pass this either a path or a file object.
3471 def __init__ (self, file, flush=1, mode='a'):
3472 if type(file) == type(''):
3474 self.file = sys.stdout
3476 self.file = open (file, mode)
3479 self.do_flush = flush
3481 def __repr__ (self):
3482 return '<file logger: %s>' % self.file
3484 def write (self, data):
3485 self.file.write (data)
3488 def writeline (self, line):
3489 self.file.writeline (line)
3492 def writelines (self, lines):
3493 self.file.writelines (lines)
3496 def maybe_flush (self):
3503 def softspace (self, *args):
3506 def log (self, message):
3507 if message[-1] not in ('\r', '\n'):
3508 self.write (message + '\n')
3510 self.write (message)
3512 def debug(self, message):
3515 class unresolving_logger:
3516 "Just in case you don't want to resolve"
3517 def __init__ (self, logger):
3518 self.logger = logger
3520 def log (self, ip, message):
3521 self.logger.log ('%s:%s' % (ip, message))
3524 def strip_eol (line):
3525 while line and line[-1] in '\r\n':
3529 VERSION_STRING = string.split(RCS_ID)[2]
3530 ATHANA_VERSION = "0.2.1"
3532 # ===========================================================================
3534 # ===========================================================================
3538 # default reply code
3541 request_counter = counter()
3543 # Whether to automatically use chunked encoding when
3545 # HTTP version is 1.1
3546 # Content-Length is not set
3547 # Chunked encoding is not already in effect
3549 # If your clients are having trouble, you might want to disable this.
3552 # by default, this request object ignores user data.
3555 def __init__ (self, *args):
3556 # unpack information about the request
3557 (self.channel, self.request,
3558 self.command, self.uri, self.version,
3562 self.reply_headers = {
3563 'Server' : 'Athana/%s' % ATHANA_VERSION,
3564 'Date' : build_http_date (time.time()),
3565 'Expires' : build_http_date (time.time())
3567 self.request_number = http_request.request_counter.increment()
3568 self._split_uri = None
3569 self._header_cache = {}
3571 # --------------------------------------------------
3572 # reply header management
3573 # --------------------------------------------------
3574 def __setitem__ (self, key, value):
3576 if key=='Set-Cookie':
3577 self.reply_headers[key] += [value]
3579 self.reply_headers[key] = [value]
3581 self.reply_headers[key] = [value]
3583 def __getitem__ (self, key):
3584 return self.reply_headers[key][0]
3586 def has_key (self, key):
3587 return self.reply_headers.has_key(key)
3589 def build_reply_header (self):
3591 for k,vv in self.reply_headers.items():
3592 if type(vv) != type([]):
3593 h += ["%s: %s" % (k,vv)]
3596 h += ["%s: %s" % (k,v)]
3597 return string.join([self.response(self.reply_code)] + h, '\r\n') + '\r\n\r\n'
3599 # --------------------------------------------------
3601 # --------------------------------------------------
3603 # <path>;<params>?<query>#<fragment>
3604 path_regex = re.compile (
3605 # path params query fragment
3606 r'([^;?#]*)(;[^?#]*)?(\?[^#]*)?(#.*)?'
3609 def split_uri (self):
3610 if self._split_uri is None:
3611 m = self.path_regex.match (self.uri)
3612 if m.end() != len(self.uri):
3613 raise ValueError, "Broken URI"
3615 self._split_uri = m.groups()
3616 return self._split_uri
3618 def get_header_with_regex (self, head_reg, group):
3619 for line in self.header:
3620 m = head_reg.match (line)
3621 if m.end() == len(line):
3622 return m.group (group)
3625 def get_header (self, header):
3626 header = string.lower (header)
3627 hc = self._header_cache
3628 if not hc.has_key (header):
3631 for line in self.header:
3632 if string.lower (line[:hl]) == h:
3641 # --------------------------------------------------
3643 # --------------------------------------------------
3645 def collect_incoming_data (self, data):
3647 self.collector.collect_incoming_data (data)
3650 'Dropping %d bytes of incoming request data' % len(data),
3654 def found_terminator (self):
3656 self.collector.found_terminator()
3659 'Unexpected end-of-record for incoming request',
3663 def push (self, thing):
3664 if type(thing) == type(''):
3665 self.outgoing.append(simple_producer (thing))
3668 self.outgoing.append(thing)
3670 def response (self, code=200):
3671 message = self.responses[code]
3672 self.reply_code = code
3673 return 'HTTP/%s %d %s' % (self.version, code, message)
3675 def error (self, code, s=None):
3676 self.reply_code = code
3678 message = self.responses[code]
3680 s = self.DEFAULT_ERROR_MESSAGE % {
3684 self['Content-Length'] = len(s)
3685 self['Content-Type'] = 'text/html'
3686 # make an error reply
3690 # can also be used for empty replies
3694 "finalize this transaction - send output to the http channel"
3696 if hasattr(self,"tempfiles"):
3697 for f in self.tempfiles:
3700 # ----------------------------------------
3701 # persistent connection management
3702 # ----------------------------------------
3704 # --- BUCKLE UP! ----
3706 connection = string.lower (get_header (CONNECTION, self.header))
3709 wrap_in_chunking = 0
3711 if self.version == '1.0':
3712 if connection == 'keep-alive':
3713 if not self.has_key ('Content-Length'):
3716 self['Connection'] = 'Keep-Alive'
3719 elif self.version == '1.1':
3720 if connection == 'close':
3722 elif not self.has_key ('Content-Length'):
3723 if self.has_key ('Transfer-Encoding'):
3724 if not self['Transfer-Encoding'] == 'chunked':
3726 elif self.use_chunked:
3727 self['Transfer-Encoding'] = 'chunked'
3728 wrap_in_chunking = 1
3731 elif self.version is None:
3732 # Although we don't *really* support http/0.9 (because we'd have to
3733 # use \r\n as a terminator, and it would just yuck up a lot of stuff)
3734 # it's very common for developers to not want to type a version number
3735 # when using telnet to debug a server.
3738 outgoing_header = simple_producer (self.build_reply_header())
3741 self['Connection'] = 'close'
3743 if wrap_in_chunking:
3744 outgoing_producer = chunked_producer (
3745 composite_producer (list(self.outgoing))
3747 # prepend the header
3748 outgoing_producer = composite_producer(
3749 [outgoing_header, outgoing_producer]
3752 # prepend the header
3753 self.outgoing.insert(0, outgoing_header)
3754 outgoing_producer = composite_producer (list(self.outgoing))
3756 # actually, this is already set to None by the handler:
3757 self.channel.current_request = None
3759 # apply a few final transformations to the output
3760 self.channel.push_with_producer (
3761 # globbing gives us large packets
3768 self.channel.close_when_done()
3770 def log_date_string (self, when):
3771 t = time.localtime(when)
3772 return time.strftime ( '%d/%b/%Y:%H:%M:%S ', t)
3775 self.channel.server.logger.log (
3776 self.channel.addr[0],
3777 '%d - - [%s] "%s"\n' % (
3778 self.channel.addr[1],
3779 self.log_date_string (time.time()),
3784 def write(self,text):
3785 if type(text) == type(''):
3787 elif type(text) == type(u''):
3788 self.push(text.encode("utf-8"))
3793 def setStatus(self,status):
3794 self.reply_code = status
3796 def makeLink(self,page,params=None):
3798 if params is not None:
3800 for k,v in params.items():
3805 query += urllib.quote(k)+"="+urllib.quote(v)
3807 return page+";"+self.sessionid+query
3809 def sendFile(self,path,content_type,force=0):
3812 file_length = os.stat(path)[stat.ST_SIZE]
3817 ims = get_header_match (IF_MODIFIED_SINCE, self.header)
3820 length = ims.group (4)
3823 length = string.atoi (length)
3824 if length != file_length:
3830 ims_date = parse_http_date (ims.group (1))
3833 mtime = os.stat (path)[stat.ST_MTIME]
3837 if length_match and ims_date:
3838 if mtime <= ims_date and not force:
3839 print "File "+path+" was not modified since "+str(ims_date)+" (current filedate is "+str(mtime)+")-> 304"
3840 self.reply_code = 304
3843 file = open (path, 'rb')
3849 self.reply_headers['Last-Modified'] = build_http_date (mtime)
3850 self.reply_headers['Content-Length'] = file_length
3851 self.reply_headers['Content-Type'] = content_type
3852 self.reply_headers['Connection'] = 'close';
3853 if self.command == 'GET':
3854 self.push(file_producer(file))
3857 def setCookie(self, name, value, expire=None):
3861 datestr = time.strftime("%a, %d-%b-%Y %H:%M:%S GMT", time.gmtime(expire))
3862 s = name+'='+value+'; expires='+datestr; #+'; path=PATH; domain=DOMAIN_NAME; secure';
3864 if 'Set-Cookie' not in self.reply_headers:
3865 self.reply_headers['Set-Cookie'] = [s]
3867 self.reply_headers['Set-Cookie'] += [s]
3869 def makeSelfLink(self,params):
3870 params2 = self.params.copy()
3871 for k,v in params.items():
3877 ret = self.makeLink(self.fullpath, params2)
3880 def writeTAL(self,page,context,macro=None):
3881 runTAL(self, context, file=page, macro=macro, request=self)
3883 def writeTALstr(self,string,context,macro=None):
3884 runTAL(self, context, string=string, macro=macro, request=self)
3886 def getTAL(self,page,context,macro=None):
3887 return processTAL(context,file=page, macro=macro, request=self)
3889 def getTALstr(self,string,context,macro=None):
3890 return processTAL(context,string=string, macro=macro, request=self)
3895 101: "Switching Protocols",
3899 203: "Non-Authoritative Information",
3901 205: "Reset Content",
3902 206: "Partial Content",
3903 300: "Multiple Choices",
3904 301: "Moved Permanently",
3905 302: "Moved Temporarily",
3907 304: "Not Modified",
3910 401: "Unauthorized",
3911 402: "Payment Required",
3914 405: "Method Not Allowed",
3915 406: "Not Acceptable",
3916 407: "Proxy Authentication Required",
3917 408: "Request Time-out",
3920 411: "Length Required",
3921 412: "Precondition Failed",
3922 413: "Request Entity Too Large",
3923 414: "Request-URI Too Large",
3924 415: "Unsupported Media Type",
3925 500: "Internal Server Error",
3926 501: "Not Implemented",
3928 503: "Service Unavailable",
3929 504: "Gateway Time-out",
3930 505: "HTTP Version not supported"
3933 # Default error message
3934 DEFAULT_ERROR_MESSAGE = string.join (
3936 '<title>Error response</title>',
3939 '<h1>Error response</h1>',
3940 '<p>Error code %(code)d.</p>',
3941 '<p>Message: %(message)s.</p>',
3948 def getTAL(page,context,macro=None,language=None):
3949 return processTAL(context,file=page, macro=macro, language=language)
3951 def getTALstr(string,context,macro=None,language=None):
3952 return processTAL(context,string=string, macro=macro, language=language)
3954 # ===========================================================================
3955 # HTTP Channel Object
3956 # ===========================================================================
3958 class http_channel (async_chat):
3960 # use a larger default output buffer
3961 ac_out_buffer_size = 1<<16
3963 current_request = None
3964 channel_counter = counter()
3966 def __init__ (self, server, conn, addr):
3967 self.channel_number = http_channel.channel_counter.increment()
3968 self.request_counter = counter()
3969 async_chat.__init__ (self, conn)
3970 self.server = server
3972 self.set_terminator ('\r\n\r\n')
3974 self.creation_time = int (time.time())
3975 self.check_maintenance()
3976 self.producer_lock = thread.allocate_lock()
3978 def initiate_send (self):
3979 self.producer_lock.acquire()
3981 async_chat.initiate_send(self)
3983 self.producer_lock.release()
3985 def push (self, data):
3987 self.producer_lock.acquire()
3989 self.producer_fifo.push (simple_producer (data))
3991 self.producer_lock.release()
3992 self.initiate_send()
3994 def push_with_producer (self, producer):
3995 self.producer_lock.acquire()
3997 self.producer_fifo.push (producer)
3999 self.producer_lock.release()
4000 self.initiate_send()
4002 def close_when_done (self):
4003 self.producer_lock.acquire()
4005 self.producer_fifo.push (None)
4007 self.producer_lock.release()
4009 #results in select.error: (9, 'Bad file descriptor') if the socket map is poll'ed
4010 #while this socket is being closed
4011 #we do it anyway, and catch the select.error in the main loop
4013 #XXX on Ubuntu's 2.6.10-5-386, the socket won't be closed until the select finishes (or
4014 #times out). We probably need to send a SIGINT signal or something. For now, we just
4015 #set a very small timeout (0.01) in the main loop, so that select() will be called often
4018 #it also results in a "NoneType has no attribute more" error if refill_buffer tries
4019 #to run data = p.more() on the None terminator (which we catch)
4021 self.initiate_send()
4022 except AttributeError:
4025 def __repr__ (self):
4026 ar = async_chat.__repr__(self)[1:-1]
4027 return '<%s channel#: %s requests:%s>' % (
4029 self.channel_number,
4030 self.request_counter
4033 # Channel Counter, Maintenance Interval...
4034 maintenance_interval = 500
4036 def check_maintenance (self):
4037 if not self.channel_number % self.maintenance_interval:
4040 def maintenance (self):
4043 # 30-minute zombie timeout. status_handler also knows how to kill zombies.
4044 zombie_timeout = 30 * 60
4046 def kill_zombies (self):
4047 now = int (time.time())
4048 for channel in asyncore.socket_map.values():
4049 if channel.__class__ == self.__class__:
4050 if (now - channel.creation_time) > channel.zombie_timeout:
4053 # --------------------------------------------------
4054 # send/recv overrides, good place for instrumentation.
4055 # --------------------------------------------------
4057 # this information needs to get into the request object,
4058 # so that it may log correctly.
4059 def send (self, data):
4060 result = async_chat.send (self, data)
4061 self.server.bytes_out.increment (len(data))
4064 def recv (self, buffer_size):
4066 result = async_chat.recv (self, buffer_size)
4067 self.server.bytes_in.increment (len(result))
4070 # --- Save a Trip to Your Service Provider ---
4071 # It's possible for a process to eat up all the memory of
4072 # the machine, and put it in an extremely wedged state,
4073 # where medusa keeps running and can't be shut down. This
4074 # is where MemoryError tends to get thrown, though of
4075 # course it could get thrown elsewhere.
4076 sys.exit ("Out of Memory!")
4078 def handle_error (self):
4079 t, v = sys.exc_info()[:2]
4083 async_chat.handle_error (self)
4085 def log (self, *args):
4088 # --------------------------------------------------
4089 # async_chat methods
4090 # --------------------------------------------------
4092 def collect_incoming_data (self, data):
4093 if self.current_request:
4094 # we are receiving data (probably POST data) for a request
4095 self.current_request.collect_incoming_data (data)
4097 # we are receiving header (request) data
4098 self.in_buffer = self.in_buffer + data
4100 def found_terminator (self):
4101 if self.current_request:
4102 self.current_request.found_terminator()
4104 header = self.in_buffer
4106 lines = string.split (header, '\r\n')
4108 # --------------------------------------------------
4109 # crack the request header
4110 # --------------------------------------------------
4112 while lines and not lines[0]:
4113 # as per the suggestion of http-1.1 section 4.1, (and
4114 # Eric Parker <eparker@zyvex.com>), ignore a leading
4115 # blank lines (buggy browsers tack it onto the end of
4120 self.close_when_done()
4125 command, uri, version = crack_request (request)
4126 header = join_headers (lines[1:])
4128 # unquote path if necessary (thanks to Skip Montanaro for pointing
4129 # out that we must unquote in piecemeal fashion).
4130 rpath, rquery = splitquery(uri)
4133 uri = unquote (rpath) + '?' + rquery
4135 uri = unquote (rpath)
4137 r = http_request (self, request, command, uri, version, header)
4138 self.request_counter.increment()
4139 self.server.total_requests.increment()
4142 self.log_info ('Bad HTTP request: %s' % repr(request), 'error')
4146 # --------------------------------------------------
4147 # handler selection and dispatch
4148 # --------------------------------------------------
4149 for h in self.server.handlers:
4152 self.current_request = r
4153 # This isn't used anywhere.
4154 # r.handler = h # CYCLE
4155 h.handle_request (r)
4157 self.server.exceptions.increment()
4158 (file, fun, line), t, v, tbinfo = asyncore.compact_traceback()
4160 'Server Error: %s, %s: file: %s line: %s' % (t,v,file,line),
4168 # no handlers, so complain
4171 # ===========================================================================
4172 # HTTP Server Object
4173 # ===========================================================================
4175 class http_server (asyncore.dispatcher):
4177 SERVER_IDENT = 'HTTP Server (V%s)' % VERSION_STRING
4179 channel_class = http_channel
4181 def __init__ (self, ip, port, resolver=None, logger_object=None):
4184 asyncore.dispatcher.__init__ (self)
4185 self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
4189 if not logger_object:
4190 logger_object = file_logger (sys.stdout)
4192 self.set_reuse_addr()
4193 self.bind ((ip, port))
4195 # lower this to 5 if your OS complains
4198 host, port = self.socket.getsockname()
4200 self.log_info('Computing default hostname', 'warning')
4201 ip = socket.gethostbyname (socket.gethostname())
4203 self.server_name = socket.gethostbyaddr (ip)[0]
4204 except socket.error:
4205 self.log_info('Cannot do reverse lookup', 'warning')
4206 self.server_name = ip # use the IP address as the "hostname"
4208 self.server_port = port
4209 self.total_clients = counter()
4210 self.total_requests = counter()
4211 self.exceptions = counter()
4212 self.bytes_out = counter()
4213 self.bytes_in = counter()
4215 if not logger_object:
4216 logger_object = file_logger (sys.stdout)
4218 self.logger = unresolving_logger (logger_object)
4221 'Athana (%s) started at %s'
4223 'The server is running! You can now direct your browser to:\n'
4227 time.ctime(time.time()),
4233 def writable (self):
4236 def handle_read (self):
4239 def readable (self):
4240 return self.accepting
4242 def handle_connect (self):
4245 def handle_accept (self):
4246 self.total_clients.increment()
4248 conn, addr = self.accept()
4249 except socket.error:
4250 # linux: on rare occasions we get a bogus socket back from
4251 # accept. socketmodule.c:makesockaddr complains that the
4252 # address family is unknown. We don't want the whole server
4253 # to shut down because of this.
4254 self.log_info ('warning: server accept() threw an exception', 'warning')
4257 # unpack non-sequence. this can happen when a read event
4258 # fires on a listening socket, but when we call accept()
4259 # we get EWOULDBLOCK, so dispatcher.accept() returns None.
4261 self.log_info ('warning: server accept() threw EWOULDBLOCK', 'warning')
4264 self.channel_class (self, conn, addr)
4266 def install_handler (self, handler, back=0):
4268 self.handlers.append (handler)
4270 self.handlers.insert (0, handler)
4272 def remove_handler (self, handler):
4273 self.handlers.remove (handler)
4276 CONNECTION = re.compile ('Connection: (.*)', re.IGNORECASE)
4278 # merge multi-line headers
4279 # [486dx2: ~500/sec]
4280 def join_headers (headers):
4282 for i in range(len(headers)):
4283 if headers[i][0] in ' \t':
4284 r[-1] = r[-1] + headers[i][1:]
4286 r.append (headers[i])
4289 def get_header (head_reg, lines, group=1):
4291 m = head_reg.match (line)
4292 if m and m.end() == len(line):
4293 return m.group (group)
4296 def get_header_match (head_reg, lines):
4298 m = head_reg.match (line)
4299 if m and m.end() == len(line):
4303 REQUEST = re.compile ('([^ ]+) ([^ ]+)(( HTTP/([0-9.]+))$|$)')
4305 def crack_request (r):
4306 m = REQUEST.match (r)
4307 if m and m.end() == len(r):
4309 version = m.group(5)
4312 return m.group(1), m.group(2), version
4314 return None, None, None
4317 # This is the 'default' handler. it implements the base set of
4318 # features expected of a simple file-delivering HTTP server. file
4319 # services are provided through a 'filesystem' object, the very same
4320 # one used by the FTP server.
4322 # You can replace or modify this handler if you want a non-standard
4323 # HTTP server. You can also derive your own handler classes from
4326 # support for handling POST requests is available in the derived
4327 # class <default_with_post_handler>, defined below.
4330 class default_handler:
4332 valid_commands = ['GET', 'HEAD']
4334 IDENT = 'Default HTTP Request Handler'
4336 # Pathnames that are tried when a URI resolves to a directory name
4337 directory_defaults = [
4342 default_file_producer = file_producer
4344 def __init__ (self, filesystem):
4345 self.filesystem = filesystem
4347 self.hit_counter = counter()
4348 # count file deliveries
4349 self.file_counter = counter()
4351 self.cache_counter = counter()
4355 def __repr__ (self):
4356 return '<%s (%s hits) at %x>' % (
4362 # always match, since this is a default
4363 def match (self, request):
4366 def can_handle(self, request):
4367 path, params, query, fragment = request.split_uri()
4369 path = unquote (path)
4370 while path and path[0] == '/':
4372 if self.filesystem.isdir (path):
4373 if path and path[-1] != '/':
4376 if path and path[-1] != '/':
4378 for default in self.directory_defaults:
4380 if self.filesystem.isfile (p):
4386 elif not self.filesystem.isfile (path):
4390 # handle a file request, with caching.
4392 def handle_request (self, request):
4394 if request.command not in self.valid_commands:
4395 request.error (400) # bad request
4398 self.hit_counter.increment()
4400 path, params, query, fragment = request.split_uri()
4403 path = unquote (path)
4405 # strip off all leading slashes
4406 while path and path[0] == '/':
4409 if self.filesystem.isdir (path):
4410 if path and path[-1] != '/':
4411 request['Location'] = 'http://%s/%s/' % (
4412 request.channel.server.server_name,
4418 # we could also generate a directory listing here,
4419 # may want to move this into another method for that
4422 if path and path[-1] != '/':
4424 for default in self.directory_defaults:
4426 if self.filesystem.isfile (p):
4431 request.error (404) # Not Found
4434 elif not self.filesystem.isfile (path):
4435 request.error (404) # Not Found
4438 file_length = self.filesystem.stat (path)[stat.ST_SIZE]
4440 ims = get_header_match (IF_MODIFIED_SINCE, request.header)
4444 length = ims.group (4)
4447 length = string.atoi (length)
4448 if length != file_length:
4456 ims_date = parse_http_date (ims.group (1))
4459 mtime = self.filesystem.stat (path)[stat.ST_MTIME]
4464 if length_match and ims_date:
4465 if mtime <= ims_date:
4466 request.reply_code = 304
4468 self.cache_counter.increment()
4469 print "File "+path+" was not modified since "+str(ims_date)+" (current filedate is "+str(mtime)+")"
4472 file = self.filesystem.open (path, 'rb')
4477 request['Last-Modified'] = build_http_date (mtime)
4478 request['Content-Length'] = file_length
4479 self.set_content_type (path, request)
4481 if request.command == 'GET':
4482 request.push (self.default_file_producer (file))
4484 self.file_counter.increment()
4487 def set_content_type (self, path, request):
4488 ext = string.lower (get_extension (path))
4489 typ, encoding = mimetypes.guess_type(path)
4491 request['Content-Type'] = typ
4493 # TODO: test a chunk off the front of the file for 8-bit
4494 # characters, and use application/octet-stream instead.
4495 request['Content-Type'] = 'text/plain'
4498 return simple_producer (
4499 '<li>%s' % html_repr (self)
4501 + ' <li><b>Total Hits:</b> %s' % self.hit_counter
4502 + ' <li><b>Files Delivered:</b> %s' % self.file_counter
4503 + ' <li><b>Cache Hits:</b> %s' % self.cache_counter
4507 # HTTP/1.0 doesn't say anything about the "; length=nnnn" addition
4508 # to this header. I suppose its purpose is to avoid the overhead
4509 # of parsing dates...
4510 IF_MODIFIED_SINCE = re.compile (
4511 'If-Modified-Since: ([^;]+)((; length=([0-9]+)$)|$)',
4515 USER_AGENT = re.compile ('User-Agent: (.*)', re.IGNORECASE)
4517 CONTENT_TYPE = re.compile (
4518 r'Content-Type: ([^;]+)((; boundary=([A-Za-z0-9\'\(\)+_,./:=?-]+)$)|$)',
4522 get_header = get_header
4523 get_header_match = get_header_match
4525 def get_extension (path):
4526 dirsep = string.rfind (path, '/')
4527 dotsep = string.rfind (path, '.')
4529 return path[dotsep+1:]
4533 class abstract_filesystem:
4534 def __init__ (self):
4537 def current_directory (self):
4538 "Return a string representing the current directory."
4541 def listdir (self, path, long=0):
4542 """Return a listing of the directory at 'path' The empty string
4543 indicates the current directory. If 'long' is set, instead
4544 return a list of (name, stat_info) tuples
4548 def open (self, path, mode):
4549 "Return an open file object"
4552 def stat (self, path):
4553 "Return the equivalent of os.stat() on the given path."
4556 def isdir (self, path):
4557 "Does the path represent a directory?"
4560 def isfile (self, path):
4561 "Does the path represent a plain file?"
4564 def cwd (self, path):
4565 "Change the working directory."
4569 "Change to the parent of the current directory."
4573 def longify (self, path):
4574 """Return a 'long' representation of the filename
4575 [for the output of the LIST command]"""
4578 # standard wrapper around a unix-like filesystem, with a 'false root'
4581 # security considerations: can symbolic links be used to 'escape' the
4582 # root? should we allow it? if not, then we could scan the
4583 # filesystem on startup, but that would not help if they were added
4584 # later. We will probably need to check for symlinks in the cwd method.
4586 # what to do if wd is an invalid directory?
4588 def safe_stat (path):
4590 return (path, os.stat (path))
4594 class os_filesystem:
4595 path_module = os.path
4597 # set this to zero if you want to disable pathname globbing.
4598 # [we currently don't glob, anyway]
4601 def __init__ (self, root, wd='/'):
4605 def current_directory (self):
4608 def isfile (self, path):
4609 p = self.normalize (self.path_module.join (self.wd, path))
4610 return self.path_module.isfile (self.translate(p))
4612 def isdir (self, path):
4613 p = self.normalize (self.path_module.join (self.wd, path))
4614 return self.path_module.isdir (self.translate(p))
4616 def cwd (self, path):
4617 p = self.normalize (self.path_module.join (self.wd, path))
4618 translated_path = self.translate(p)
4619 if not self.path_module.isdir (translated_path):
4622 old_dir = os.getcwd()
4623 # temporarily change to that directory, in order
4624 # to see if we have permission to do so.
4628 os.chdir (translated_path)
4639 return self.cwd ('..')
4641 def listdir (self, path, long=0):
4642 p = self.translate (path)
4643 # I think we should glob, but limit it to the current
4647 return list_producer (ld, None)
4649 old_dir = os.getcwd()
4652 # if os.stat fails we ignore that file.
4653 result = filter (None, map (safe_stat, ld))
4656 return list_producer (result, self.longify)
4658 # TODO: implement a cache w/timeout for stat()
4659 def stat (self, path):
4660 p = self.translate (path)
4663 def open (self, path, mode):
4664 p = self.translate (path)
4665 return open (p, mode)
4667 def unlink (self, path):
4668 p = self.translate (path)
4669 return os.unlink (p)
4671 def mkdir (self, path):
4672 p = self.translate (path)
4675 def rmdir (self, path):
4676 p = self.translate (path)
4680 def normalize (self, path):
4681 # watch for the ever-sneaky '/+' path element
4682 path = re.sub('/+', '/', path)
4683 p = self.path_module.normpath (path)
4684 # remove 'dangling' cdup's.
4685 if len(p) > 2 and p[:3] == '/..':
4689 def translate (self, path):
4690 # we need to join together three separate
4691 # path components, and do it safely.
4692 # <real_root>/<current_directory>/<path>
4693 # use the operating system's path separator.
4694 path = string.join (string.split (path, '/'), os.sep)
4695 p = self.normalize (self.path_module.join (self.wd, path))
4696 p = self.normalize (self.path_module.join (self.root, p[1:]))
4699 def longify (self, (path, stat_info)):
4700 return unix_longify (path, stat_info)
4702 def __repr__ (self):
4703 return '<unix-style fs root:%s wd:%s>' % (
4708 # this matches the output of NT's ftp server (when in
4709 # MSDOS mode) exactly.
4711 def msdos_longify (file, stat_info):
4712 if stat.S_ISDIR (stat_info[stat.ST_MODE]):
4716 date = msdos_date (stat_info[stat.ST_MTIME])
4717 return '%s %s %8d %s' % (
4720 stat_info[stat.ST_SIZE],
4726 info = time.gmtime (t)
4728 info = time.gmtime (0)
4729 # year, month, day, hour, minute, second, ...
4732 info[3] = info[3] - 12
4735 return '%02d-%02d-%02d %02d:%02d%s' % (
4744 months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
4745 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
4758 def unix_longify (file, stat_info):
4759 # for now, only pay attention to the lower bits
4760 mode = ('%o' % stat_info[stat.ST_MODE])[-3:]
4761 mode = string.join (map (lambda x: mode_table[x], mode), '')
4762 if stat.S_ISDIR (stat_info[stat.ST_MODE]):
4766 date = ls_date (long(time.time()), stat_info[stat.ST_MTIME])
4767 return '%s%s %3d %-8d %-8d %8d %s %s' % (
4770 stat_info[stat.ST_NLINK],
4771 stat_info[stat.ST_UID],
4772 stat_info[stat.ST_GID],
4773 stat_info[stat.ST_SIZE],
4778 # Emulate the unix 'ls' command's date field.
4779 # it has two formats - if the date is more than 180
4780 # days in the past, then it's like this:
4782 # otherwise, it looks like this:
4785 def ls_date (now, t):
4787 info = time.gmtime (t)
4789 info = time.gmtime (0)
4790 # 15,600,000 == 86,400 * 180
4791 if (now - t) > 15600000:
4792 return '%s %2d %d' % (
4798 return '%s %2d %02d:%02d' % (
4805 class list_producer:
4806 def __init__ (self, list, func=None):
4810 # this should do a pushd/popd
4815 # do a few at a time
4816 bunch = self.list[:50]
4817 if self.func is not None:
4818 bunch = map (self.func, bunch)
4819 self.list = self.list[50:]
4820 return string.joinfields (bunch, '\r\n') + '\r\n'
4822 class hooked_callback:
4823 def __init__ (self, hook, callback):
4824 self.hook, self.callback = hook, callback
4826 def __call__ (self, *args):
4827 apply (self.hook, args)
4828 apply (self.callback, args)
4830 # An extensible, configurable, asynchronous FTP server.
4832 # All socket I/O is non-blocking, however file I/O is currently
4833 # blocking. Eventually file I/O may be made non-blocking, too, if it
4834 # seems necessary. Currently the only CPU-intensive operation is
4835 # getting and formatting a directory listing. [this could be moved
4836 # into another process/directory server, or another thread?]
4838 # Only a subset of RFC 959 is implemented, but much of that RFC is
4839 # vestigial anyway. I've attempted to include the most commonly-used
4840 # commands, using the feature set of wu-ftpd as a guide.
4843 # TODO: implement a directory listing cache. On very-high-load
4844 # servers this could save a lot of disk abuse, and possibly the
4845 # work of computing emulated unix ls output.
4847 # Potential security problem with the FTP protocol? I don't think
4848 # there's any verification of the origin of a data connection. Not
4849 # really a problem for the server (since it doesn't send the port
4850 # command, except when in PASV mode) But I think a data connection
4851 # could be spoofed by a program with access to a sniffer - it could
4852 # watch for a PORT command to go over a command channel, and then
4853 # connect to that port before the server does.
4856 # In order to support assuming the id of a particular user,
4857 # it seems there are two options:
4858 # 1) fork, and seteuid in the child
4859 # 2) carefully control the effective uid around filesystem accessing
4860 # methods, using try/finally. [this seems to work]
4862 VERSION = string.split(RCS_ID)[2]
4864 class ftp_channel (async_chat):
4866 # defaults for a reliable __repr__
4867 addr = ('unknown','0')
4869 # unset this in a derived class in order
4870 # to enable the commands in 'self.write_commands'
4872 write_commands = ['appe','dele','mkd','rmd','rnfr','rnto','stor','stou']
4874 restart_position = 0
4876 # comply with (possibly troublesome) RFC959 requirements
4877 # This is necessary to correctly run an active data connection
4878 # through a firewall that triggers on the source port (expected
4879 # to be 'L-1', or 20 in the normal case).
4880 bind_local_minus_one = 0
4882 def __init__ (self, server, conn, addr):
4883 self.server = server
4884 self.current_mode = 'a'
4886 async_chat.__init__ (self, conn)
4887 self.set_terminator ('\r\n')
4889 # client data port. Defaults to 'the same as the control connection'.
4890 self.client_addr = (addr[0], 21)
4892 self.client_dc = None
4895 self.passive_acceptor = None
4896 self.passive_connection = None
4897 self.filesystem = None
4901 '220 %s FTP server (Medusa Async V%s [experimental]) ready.' % (
4902 self.server.hostname,
4907 # def __del__ (self):
4908 # print 'ftp_channel.__del__()'
4910 # --------------------------------------------------
4911 # async-library methods
4912 # --------------------------------------------------
4914 def handle_expt (self):
4915 # this is handled below. not sure what I could
4916 # do here to make that code less kludgish.
4919 def collect_incoming_data (self, data):
4920 self.in_buffer = self.in_buffer + data
4921 if len(self.in_buffer) > 4096:
4922 # silently truncate really long lines
4923 # (possible denial-of-service attack)
4926 def found_terminator (self):
4928 line = self.in_buffer
4933 sp = string.find (line, ' ')
4935 line = [line[:sp], line[sp+1:]]
4939 command = string.lower (line[0])
4940 # watch especially for 'urgent' abort commands.
4941 if string.find (command, 'abor') != -1:
4942 # strip off telnet sync chars and the like...
4943 while command and command[0] not in string.letters:
4944 command = command[1:]
4945 fun_name = 'cmd_%s' % command
4946 if command != 'pass':
4947 self.log ('<== %s' % repr(self.in_buffer)[1:-1])
4949 self.log ('<== %s' % line[0]+' <password>')
4951 if not hasattr (self, fun_name):
4952 self.command_not_understood (line[0])
4954 fun = getattr (self, fun_name)
4955 if (not self.authorized) and (command not in ('user', 'pass', 'help', 'quit')):
4956 self.respond ('530 Please log in with USER and PASS')
4957 elif (not self.check_command_authorization (command)):
4958 self.command_not_authorized (command)
4961 result = apply (fun, (line,))
4963 self.server.total_exceptions.increment()
4964 (file, fun, line), t,v, tbinfo = asyncore.compact_traceback()
4967 self.client_dc.close()
4971 '451 Server Error: %s, %s: file: %s line: %s' % (
4980 if self.passive_acceptor:
4981 self.passive_acceptor.close()
4983 self.client_dc.close()
4984 self.server.closed_sessions.increment()
4985 async_chat.close (self)
4987 # --------------------------------------------------
4988 # filesystem interface functions.
4989 # override these to provide access control or perform
4991 # --------------------------------------------------
4993 def cwd (self, line):
4994 return self.filesystem.cwd (line[1])
4996 def cdup (self, line):
4997 return self.filesystem.cdup()
4999 def open (self, path, mode):
5000 return self.filesystem.open (path, mode)
5002 # returns a producer
5003 def listdir (self, path, long=0):
5004 return self.filesystem.listdir (path, long)
5006 def get_dir_list (self, line, long=0):
5007 # we need to scan the command line for arguments to '/bin/ls'...
5012 path_args.append (arg)
5016 if len(path_args) < 1:
5020 return self.listdir (dir, long)
5022 # --------------------------------------------------
5023 # authorization methods
5024 # --------------------------------------------------
5026 def check_command_authorization (self, command):
5027 if command in self.write_commands and self.read_only:
5032 # --------------------------------------------------
5034 # --------------------------------------------------
5036 def log (self, message):
5037 self.server.logger.log (
5040 self.addr[1], message
5044 def respond (self, resp):
5045 self.log ('==> %s' % resp)
5046 self.push (resp + '\r\n')
5048 def command_not_understood (self, command):
5049 self.respond ("500 '%s': command not understood." % command)
5051 def command_not_authorized (self, command):
5053 "530 You are not authorized to perform the '%s' command" % (
5058 def make_xmit_channel (self):
5059 # In PASV mode, the connection may or may _not_ have been made
5060 # yet. [although in most cases it is... FTP Explorer being
5061 # the only exception I've yet seen]. This gets somewhat confusing
5062 # because things may happen in any order...
5063 pa = self.passive_acceptor
5066 # a connection has already been made.
5067 conn, addr = self.passive_acceptor.ready
5068 cdc = xmit_channel (self, addr)
5069 cdc.set_socket (conn)
5071 self.passive_acceptor.close()
5072 self.passive_acceptor = None
5074 # we're still waiting for a connect to the PASV port.
5075 cdc = xmit_channel (self)
5078 ip, port = self.client_addr
5079 cdc = xmit_channel (self, self.client_addr)
5080 cdc.create_socket (socket.AF_INET, socket.SOCK_STREAM)
5081 if self.bind_local_minus_one:
5082 cdc.bind (('', self.server.port - 1))
5084 cdc.connect ((ip, port))
5085 except socket.error, why:
5086 self.respond ("425 Can't build data connection")
5087 self.client_dc = cdc
5089 # pretty much the same as xmit, but only right on the verge of
5090 # being worth a merge.
5091 def make_recv_channel (self, fd):
5092 pa = self.passive_acceptor
5095 # a connection has already been made.
5096 conn, addr = pa.ready
5097 cdc = recv_channel (self, addr, fd)
5098 cdc.set_socket (conn)
5100 self.passive_acceptor.close()
5101 self.passive_acceptor = None
5103 # we're still waiting for a connect to the PASV port.
5104 cdc = recv_channel (self, None, fd)
5107 ip, port = self.client_addr
5108 cdc = recv_channel (self, self.client_addr, fd)
5109 cdc.create_socket (socket.AF_INET, socket.SOCK_STREAM)
5111 cdc.connect ((ip, port))
5112 except socket.error, why:
5113 self.respond ("425 Can't build data connection")
5114 self.client_dc = cdc
5130 # --------------------------------------------------
5132 # --------------------------------------------------
5134 def cmd_type (self, line):
5135 'specify data transfer type'
5136 # ascii, ebcdic, image, local <byte size>
5137 t = string.lower (line[1])
5138 # no support for EBCDIC
5139 # if t not in ['a','e','i','l']:
5140 if t not in ['a','i','l']:
5141 self.command_not_understood (string.join (line))
5142 elif t == 'l' and (len(line) > 2 and line[2] != '8'):
5143 self.respond ('504 Byte size must be 8')
5145 self.current_mode = t
5146 self.respond ('200 Type set to %s.' % self.type_map[t])
5149 def cmd_quit (self, line):
5151 self.respond ('221 Goodbye.')
5152 self.close_when_done()
5154 def cmd_port (self, line):
5155 'specify data connection port'
5156 info = string.split (line[1], ',')
5157 ip = string.join (info[:4], '.')
5158 port = string.atoi(info[4])*256 + string.atoi(info[5])
5159 # how many data connections at a time?
5160 # I'm assuming one for now...
5161 # TODO: we should (optionally) verify that the
5162 # ip number belongs to the client. [wu-ftpd does this?]
5163 self.client_addr = (ip, port)
5164 self.respond ('200 PORT command successful.')
5166 def new_passive_acceptor (self):
5167 # ensure that only one of these exists at a time.
5168 if self.passive_acceptor is not None:
5169 self.passive_acceptor.close()
5170 self.passive_acceptor = None
5171 self.passive_acceptor = passive_acceptor (self)
5172 return self.passive_acceptor
5174 def cmd_pasv (self, line):
5175 'prepare for server-to-server transfer'
5176 pc = self.new_passive_acceptor()
5178 ip_addr = pc.control_channel.getsockname()[0]
5180 '227 Entering Passive Mode (%s,%d,%d)' % (
5181 string.replace(ip_addr, '.', ','),
5186 self.client_dc = None
5188 def cmd_nlst (self, line):
5189 'give name list of files in directory'
5190 # ncftp adds the -FC argument for the user-visible 'nlist'
5191 # command. We could try to emulate ls flags, but not just yet.
5195 dir_list_producer = self.get_dir_list (line, 0)
5196 except os.error, why:
5197 self.respond ('550 Could not list directory: %s' % why)
5200 '150 Opening %s mode data connection for file list' % (
5201 self.type_map[self.current_mode]
5204 self.make_xmit_channel()
5205 self.client_dc.push_with_producer (dir_list_producer)
5206 self.client_dc.close_when_done()
5208 def cmd_list (self, line):
5209 'give a list of files in a directory'
5211 dir_list_producer = self.get_dir_list (line, 1)
5212 except os.error, why:
5213 self.respond ('550 Could not list directory: %s' % why)
5216 '150 Opening %s mode data connection for file list' % (
5217 self.type_map[self.current_mode]
5220 self.make_xmit_channel()
5221 self.client_dc.push_with_producer (dir_list_producer)
5222 self.client_dc.close_when_done()
5224 def cmd_cwd (self, line):
5225 'change working directory'
5227 self.respond ('250 CWD command successful.')
5229 self.respond ('550 No such directory.')
5231 def cmd_cdup (self, line):
5232 'change to parent of current working directory'
5234 self.respond ('250 CDUP command successful.')
5236 self.respond ('550 No such directory.')
5238 def cmd_pwd (self, line):
5239 'print the current working directory'
5241 '257 "%s" is the current directory.' % (
5242 self.filesystem.current_directory()
5248 # 213 19960301204320
5249 def cmd_mdtm (self, line):
5250 'show last modification time of file'
5252 if not self.filesystem.isfile (filename):
5253 self.respond ('550 "%s" is not a file' % filename)
5255 mtime = time.gmtime(self.filesystem.stat(filename)[stat.ST_MTIME])
5257 '213 %4d%02d%02d%02d%02d%02d' % (
5267 def cmd_noop (self, line):
5269 self.respond ('200 NOOP command successful.')
5271 def cmd_size (self, line):
5272 'return size of file'
5274 if not self.filesystem.isfile (filename):
5275 self.respond ('550 "%s" is not a file' % filename)
5278 '213 %d' % (self.filesystem.stat(filename)[stat.ST_SIZE])
5281 def cmd_retr (self, line):
5284 self.command_not_understood (string.join (line))
5287 if not self.filesystem.isfile (file):
5288 self.log_info ('checking %s' % file)
5289 self.respond ('550 No such file')
5292 # FIXME: for some reason, 'rt' isn't working on win95
5293 mode = 'r'+self.type_mode_map[self.current_mode]
5294 fd = self.open (file, mode)
5295 except IOError, why:
5296 self.respond ('553 could not open file for reading: %s' % (repr(why)))
5299 "150 Opening %s mode data connection for file '%s'" % (
5300 self.type_map[self.current_mode],
5304 self.make_xmit_channel()
5306 if self.restart_position:
5307 # try to position the file as requested, but
5308 # give up silently on failure (the 'file object'
5309 # may not support seek())
5311 fd.seek (self.restart_position)
5314 self.restart_position = 0
5316 self.client_dc.push_with_producer (
5319 self.client_dc.close_when_done()
5321 def cmd_stor (self, line, mode='wb'):
5324 self.command_not_understood (string.join (line))
5326 if self.restart_position:
5327 restart_position = 0
5328 self.respond ('553 restart on STOR not yet supported')
5331 # todo: handle that type flag
5333 fd = self.open (file, mode)
5334 except IOError, why:
5335 self.respond ('553 could not open file for writing: %s' % (repr(why)))
5338 '150 Opening %s connection for %s' % (
5339 self.type_map[self.current_mode],
5343 self.make_recv_channel (fd)
5345 def cmd_abor (self, line):
5348 self.client_dc.close()
5349 self.respond ('226 ABOR command successful.')
5351 def cmd_appe (self, line):
5353 return self.cmd_stor (line, 'ab')
5355 def cmd_dele (self, line):
5357 self.command_not_understood (string.join (line))
5360 if self.filesystem.isfile (file):
5362 self.filesystem.unlink (file)
5363 self.respond ('250 DELE command successful.')
5365 self.respond ('550 error deleting file.')
5367 self.respond ('550 %s: No such file.' % file)
5369 def cmd_mkd (self, line):
5371 self.command_not_understood (string.join (line))
5375 self.filesystem.mkdir (path)
5376 self.respond ('257 MKD command successful.')
5378 self.respond ('550 error creating directory.')
5380 def cmd_rmd (self, line):
5382 self.command_not_understood (string.join (line))
5386 self.filesystem.rmdir (path)
5387 self.respond ('250 RMD command successful.')
5389 self.respond ('550 error removing directory.')
5391 def cmd_user (self, line):
5395 self.respond ('331 Password required.')
5397 self.command_not_understood (string.join (line))
5399 def cmd_pass (self, line):
5405 result, message, fs = self.server.authorizer.authorize (self, self.user, pw)
5407 self.respond ('230 %s' % message)
5408 self.filesystem = fs
5410 self.log_info('Successful login: Filesystem=%s' % repr(fs))
5412 self.respond ('530 %s' % message)
5414 def cmd_rest (self, line):
5415 'restart incomplete transfer'
5417 pos = string.atoi (line[1])
5419 self.command_not_understood (string.join (line))
5420 self.restart_position = pos
5422 '350 Restarting at %d. Send STORE or RETRIEVE to initiate transfer.' % pos
5425 def cmd_stru (self, line):
5426 'obsolete - set file transfer structure'
5429 self.respond ('200 STRU F Ok')
5431 self.respond ('504 Unimplemented STRU type')
5433 def cmd_mode (self, line):
5434 'obsolete - set file transfer mode'
5437 self.respond ('200 MODE S Ok')
5439 self.respond ('502 Unimplemented MODE type')
5441 # The stat command has two personalities. Normally it returns status
5442 # information about the current connection. But if given an argument,
5443 # it is equivalent to the LIST command, with the data sent over the
5444 # control connection. Strange. But wuftpd, ftpd, and nt's ftp server
5447 ## def cmd_stat (self, line):
5448 ## 'return status of server'
5451 def cmd_syst (self, line):
5452 'show operating system type of server system'
5453 # Replying to this command is of questionable utility, because
5454 # this server does not behave in a predictable way w.r.t. the
5455 # output of the LIST command. We emulate Unix ls output, but
5456 # on win32 the pathname can contain drive information at the front
5457 # Currently, the combination of ensuring that os.sep == '/'
5458 # and removing the leading slash when necessary seems to work.
5459 # [cd'ing to another drive also works]
5461 # This is how wuftpd responds, and is probably
5462 # the most expected. The main purpose of this reply is so that
5463 # the client knows to expect Unix ls-style LIST output.
5464 self.respond ('215 UNIX Type: L8')
5465 # one disadvantage to this is that some client programs
5466 # assume they can pass args to /bin/ls.
5467 # a few typical responses:
5468 # 215 UNIX Type: L8 (wuftpd)
5469 # 215 Windows_NT version 3.51
5470 # 215 VMS MultiNet V3.3
5471 # 500 'SYST': command not understood. (SVR4)
5473 def cmd_help (self, line):
5474 'give help information'
5475 # find all the methods that match 'cmd_xxxx',
5476 # use their docstrings for the help response.
5477 attrs = dir(self.__class__)
5480 if attr[:4] == 'cmd_':
5481 x = getattr (self, attr)
5482 if type(x) == type(self.cmd_help):
5484 help_lines.append ('\t%s\t%s' % (attr[4:], x.__doc__))
5486 self.push ('214-The following commands are recognized\r\n')
5487 self.push_with_producer (lines_producer (help_lines))
5488 self.push ('214\r\n')
5490 self.push ('214-\r\n\tHelp Unavailable\r\n214\r\n')
5492 class ftp_server (asyncore.dispatcher):
5493 # override this to spawn a different FTP channel class.
5494 ftp_channel_class = ftp_channel
5496 SERVER_IDENT = 'FTP Server (V%s)' % VERSION
5504 logger_object=file_logger (sys.stdout)
5508 self.authorizer = authorizer
5510 if hostname is None:
5511 self.hostname = socket.gethostname()
5513 self.hostname = hostname
5516 self.total_sessions = counter()
5517 self.closed_sessions = counter()
5518 self.total_files_out = counter()
5519 self.total_files_in = counter()
5520 self.total_bytes_out = counter()
5521 self.total_bytes_in = counter()
5522 self.total_exceptions = counter()
5524 asyncore.dispatcher.__init__ (self)
5525 self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
5527 self.set_reuse_addr()
5528 self.bind ((self.ip, self.port))
5531 if not logger_object:
5532 logger_object = sys.stdout
5534 self.logger = unresolving_logger (logger_object)
5536 self.log_info('FTP server started at %s\n\tAuthorizer:%s\n\tHostname: %s\n\tPort: %d' % (
5537 time.ctime(time.time()),
5538 repr (self.authorizer),
5543 def writable (self):
5546 def handle_read (self):
5549 def handle_connect (self):
5552 def handle_accept (self):
5553 conn, addr = self.accept()
5554 self.total_sessions.increment()
5555 self.log_info('Incoming connection from %s:%d' % (addr[0], addr[1]))
5556 self.ftp_channel_class (self, conn, addr)
5558 # return a producer describing the state of the server
5562 return string.join (english_bytes (n))
5564 return lines_producer (
5565 ['<h2>%s</h2>' % self.SERVER_IDENT,
5566 '<br>Listening on <b>Host:</b> %s' % self.hostname,
5567 '<b>Port:</b> %d' % self.port,
5569 '<b>Total:</b> %s' % self.total_sessions,
5570 '<b>Current:</b> %d' % (self.total_sessions.as_long() - self.closed_sessions.as_long()),
5572 '<b>Sent:</b> %s' % self.total_files_out,
5573 '<b>Received:</b> %s' % self.total_files_in,
5575 '<b>Sent:</b> %s' % nice_bytes (self.total_bytes_out.as_long()),
5576 '<b>Received:</b> %s' % nice_bytes (self.total_bytes_in.as_long()),
5577 '<br>Exceptions: %s' % self.total_exceptions,
5581 # ======================================================================
5582 # Data Channel Classes
5583 # ======================================================================
5585 # This socket accepts a data connection, used when the server has been
5586 # placed in passive mode. Although the RFC implies that we ought to
5587 # be able to use the same acceptor over and over again, this presents
5588 # a problem: how do we shut it off, so that we are accepting
5589 # connections only when we expect them? [we can't]
5591 # wuftpd, and probably all the other servers, solve this by allowing
5592 # only one connection to hit this acceptor. They then close it. Any
5593 # subsequent data-connection command will then try for the default
5594 # port on the client side [which is of course never there]. So the
5595 # 'always-send-PORT/PASV' behavior seems required.
5597 # Another note: wuftpd will also be listening on the channel as soon
5598 # as the PASV command is sent. It does not wait for a data command
5601 # --- we need to queue up a particular behavior:
5602 # 1) xmit : queue up producer[s]
5603 # 2) recv : the file object
5605 # It would be nice if we could make both channels the same. Hmmm..
5608 class passive_acceptor (asyncore.dispatcher):
5611 def __init__ (self, control_channel):
5612 # connect_fun (conn, addr)
5613 asyncore.dispatcher.__init__ (self)
5614 self.control_channel = control_channel
5615 self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
5616 # bind to an address on the interface that the
5617 # control connection is coming from.
5619 self.control_channel.getsockname()[0],
5622 self.addr = self.getsockname()
5625 # def __del__ (self):
5626 # print 'passive_acceptor.__del__()'
5628 def log (self, *ignore):
5631 def handle_accept (self):
5632 conn, addr = self.accept()
5633 dc = self.control_channel.client_dc
5635 dc.set_socket (conn)
5638 self.control_channel.passive_acceptor = None
5640 self.ready = conn, addr
5644 class xmit_channel (async_chat):
5646 # for an ethernet, you want this to be fairly large, in fact, it
5647 # _must_ be large for performance comparable to an ftpd. [64k] we
5648 # ought to investigate automatically-sized buffers...
5650 ac_out_buffer_size = 16384
5653 def __init__ (self, channel, client_addr=None):
5654 self.channel = channel
5655 self.client_addr = client_addr
5656 async_chat.__init__ (self)
5658 # def __del__ (self):
5659 # print 'xmit_channel.__del__()'
5661 def log (self, *args):
5664 def readable (self):
5665 return not self.connected
5667 def writable (self):
5670 def send (self, data):
5671 result = async_chat.send (self, data)
5672 self.bytes_out = self.bytes_out + result
5675 def handle_error (self):
5676 # usually this is to catch an unexpected disconnect.
5677 self.log_info ('unexpected disconnect on data xmit channel', 'error')
5683 # TODO: there's a better way to do this. we need to be able to
5684 # put 'events' in the producer fifo. to do this cleanly we need
5685 # to reposition the 'producer' fifo as an 'event' fifo.
5691 s.total_files_out.increment()
5692 s.total_bytes_out.increment (self.bytes_out)
5693 if not len(self.producer_fifo):
5694 c.respond ('226 Transfer complete')
5696 c.respond ('426 Connection closed; transfer aborted')
5700 async_chat.close (self)
5702 class recv_channel (asyncore.dispatcher):
5703 def __init__ (self, channel, client_addr, fd):
5704 self.channel = channel
5705 self.client_addr = client_addr
5707 asyncore.dispatcher.__init__ (self)
5708 self.bytes_in = counter()
5710 def log (self, *ignore):
5713 def handle_connect (self):
5716 def writable (self):
5720 result = apply (asyncore.dispatcher.recv, args)
5722 self.bytes_in.increment(len(result))
5727 def handle_read (self):
5728 block = self.recv (self.buffer_size)
5731 self.fd.write (block)
5733 self.log_info ('got exception writing block...', 'error')
5735 def handle_close (self):
5736 s = self.channel.server
5737 s.total_files_in.increment()
5738 s.total_bytes_in.increment(self.bytes_in.as_long())
5740 self.channel.respond ('226 Transfer complete.')
5759 HTTP_SWITCHING_PROTOCOLS = 101
5760 HTTP_PROCESSING = 102
5764 HTTP_NON_AUTHORITATIVE = 203
5765 HTTP_NO_CONTENT = 204
5766 HTTP_RESET_CONTENT = 205
5767 HTTP_PARTIAL_CONTENT = 206
5768 HTTP_MULTI_STATUS = 207
5769 HTTP_MULTIPLE_CHOICES = 300
5770 HTTP_MOVED_PERMANENTLY = 301
5771 HTTP_MOVED_TEMPORARILY = 302
5772 HTTP_SEE_OTHER = 303
5773 HTTP_NOT_MODIFIED = 304
5774 HTTP_USE_PROXY = 305
5775 HTTP_TEMPORARY_REDIRECT = 307
5776 HTTP_BAD_REQUEST = 400
5777 HTTP_UNAUTHORIZED = 401
5778 HTTP_PAYMENT_REQUIRED = 402
5779 HTTP_FORBIDDEN = 403
5780 HTTP_NOT_FOUND = 404
5781 HTTP_METHOD_NOT_ALLOWED = 405
5782 HTTP_NOT_ACCEPTABLE = 406
5783 HTTP_PROXY_AUTHENTICATION_REQUIRED= 407
5784 HTTP_REQUEST_TIME_OUT = 408
5787 HTTP_LENGTH_REQUIRED = 411
5788 HTTP_PRECONDITION_FAILED = 412
5789 HTTP_REQUEST_ENTITY_TOO_LARGE = 413
5790 HTTP_REQUEST_URI_TOO_LARGE = 414
5791 HTTP_UNSUPPORTED_MEDIA_TYPE = 415
5792 HTTP_RANGE_NOT_SATISFIABLE = 416
5793 HTTP_EXPECTATION_FAILED = 417
5794 HTTP_UNPROCESSABLE_ENTITY = 422
5796 HTTP_FAILED_DEPENDENCY = 424
5797 HTTP_INTERNAL_SERVER_ERROR = 500
5798 HTTP_NOT_IMPLEMENTED = 501
5799 HTTP_BAD_GATEWAY = 502
5800 HTTP_SERVICE_UNAVAILABLE = 503
5801 HTTP_GATEWAY_TIME_OUT = 504
5802 HTTP_VERSION_NOT_SUPPORTED = 505
5803 HTTP_VARIANT_ALSO_VARIES = 506
5804 HTTP_INSUFFICIENT_STORAGE = 507
5805 HTTP_NOT_EXTENDED = 510
5807 GLOBAL_TEMP_DIR="/tmp/"
5808 GLOBAL_ROOT_DIR="no-root-dir-set"
5810 multithreading_enabled = 0
5811 number_of_threads = 32
5813 def qualify_path(p):
5818 def join_paths(p1,p2):
5819 if p1.endswith("/"):
5820 if p2.startswith("/"):
5825 if p2.startswith("/"):
5828 return p1 + "/" + p2
5836 def getMacroFile(filename):
5837 global macrofile_callback
5838 for r in macroresolvers:
5841 if f is not None and os.path.isfile(f):
5845 if os.path.isfile(filename):
5847 filename2 = join_paths(GLOBAL_ROOT_DIR,filename)
5848 if os.path.isfile(filename2):
5850 raise IOError("No such file: "+filename2)
5855 def _make_inifiles(root, path):
5856 dirs = path.split("/")
5859 path = join_paths(path, dir)
5860 inifile = join_paths(path, "__init__.py")
5861 # create missing __init__.py
5862 if not os.path.isfile(inifile):
5864 lg.log("creating file "+inifile)
5865 open(inifile, "wb").close()
5867 def _load_module(filename):
5868 global global_modules
5869 b = BASENAME.match(filename)
5870 # filename e.g. /my/modules/test.py
5871 # b.group(1) = /my/modules/
5872 # b.group(2) = test.py
5874 raise "Internal error with filename "+filename
5877 raise "Internal error with filename "+filename
5879 while filename.startswith("./"):
5880 filename = filename[2:]
5882 if filename in global_modules:
5883 return global_modules[filename]
5885 dir = os.path.dirname(filename)
5886 path = dir.replace("/",".")
5888 _make_inifiles(GLOBAL_ROOT_DIR, dir)
5890 # strip tailing/leading dots
5891 while len(path) and path[0] == '.':
5893 while len(path) and path[-1] != '.':
5896 module2 = (path + module)
5898 lg.log("Loading module "+module2)
5900 m = __import__(module2)
5902 i = module2.index(".")
5903 m = eval("m."+module2[i+1:])
5904 global_modules[filename] = m
5909 system_modules = sys.modules.copy()
5910 stdlib, x = os.path.split(os.__file__)
5911 def _purge_all_modules():
5912 for m,mod in sys.modules.items():
5913 if m not in system_modules:
5914 if hasattr(mod, "__file__"):
5916 path, x = os.path.split(f)
5917 if not path.startswith(stdlib):
5921 def __init__(self, name, root=None):
5924 self.startupfile = None
5926 self.root = qualify_path(root)
5927 self.pattern_to_function = {}
5928 self.id_to_function = {}
5930 def addFile(self, filename):
5931 file = WebFile(self, filename)
5932 self.files += [file]
5935 def setRoot(self, root):
5936 self.root = qualify_path(root)
5937 while self.root.startswith("./"):
5938 self.root = self.root[2:]
5940 def setStartupFile(self, startupfile):
5941 self.startupfile = startupfile
5942 lg.log(" executing startupfile")
5943 self._load_module(self.startupfile)
5945 def getStartupFile(self):
5946 return self.startupfile
5948 def match(self, path):
5950 for pattern,call in self.pattern_to_function.items():
5951 if pattern.match(path):
5952 function,desc = call
5954 lg.log("Request %s matches (%s)" % (req.path, desc))
5955 if function is None:
5956 for id,call in self.id_to_function.items():
5958 function,desc = call
5960 lg.log("Request %s matches handler (%s)" % (req.path, desc))
5963 def call_and_close(f,req):
5965 if status is not None and type(1)==type(status) and status>10:
5966 req.reply_code = status
5968 return req.error(status, "not found")
5969 elif(status >= 400 and status <= 500):
5970 return req.error(status)
5972 return lambda req: call_and_close(function,req)
5975 def __init__(self, name, root=None):
5978 if type(root) == type(""):
5980 elif type(root) == type([]):
5984 def match(self, path):
5985 return lambda req: self.findfile(req)
5987 def findfile(self, request):
5988 for handler in self.handlers:
5989 if handler.can_handle(request):
5990 return handler.handle_request(request)
5991 return request.error(404, "File "+request.path+" not found")
5993 def addRoot(self, dir):
5994 dir = qualify_path(dir)
5995 while dir.startswith("./"):
5997 if zipfile.is_zipfile(GLOBAL_ROOT_DIR + dir[:-1]) and dir.lower().endswith("zip/"):
5998 self.handlers += [default_handler (zip_filesystem (GLOBAL_ROOT_DIR + dir[:-1]))]
6000 self.handlers += [default_handler (os_filesystem (GLOBAL_ROOT_DIR + dir))]
6003 def __init__(self, context, filename):
6004 self.context = context
6005 if filename[0] == '/':
6006 filename = filename[1:]
6007 self.filename = filename
6008 self.m = _load_module(filename)
6011 def addHandler(self, function):
6012 handler = WebHandler(self, function)
6013 self.handlers += [handler]
6016 def addFTPHandler(self, ftpclass):
6020 c = eval("m."+ftpclass)
6025 lgerr.log("Error in FTP Handler:" + str(sys.exc_info()[0]) + " " + str(sys.exc_info()[1]))
6026 traceback.print_tb(sys.exc_info()[2],None,lgerr)
6027 raise "No such function "+ftpclass+" in file "+self.filename
6029 def addMacroResolver(self, macroresolver):
6030 global macroresolvers
6033 f = eval("m."+macroresolver)
6036 macroresolvers += [f]
6038 lgerr.log("Error in Macro Resolver:" + str(sys.exc_info()[0]) + " " + str(sys.exc_info()[1]))
6039 traceback.print_tb(sys.exc_info()[2],None,lgerr)
6040 raise "No such function "+macroresolver+" in file "+self.filename
6042 def addTranslator(self, handler):
6046 f = eval("m."+translator)
6051 lgerr.log("Error in Macro Resolver:" + str(sys.exc_info()[0]) + " " + str(sys.exc_info()[1]))
6052 traceback.print_tb(sys.exc_info()[2],None,lgerr)
6053 raise "No such function "+translator+" in file "+self.filename
6055 def getFileName(self):
6056 return self.context.root + self.filename
6059 def __init__(self, file, function):
6061 self.function = function
6064 self.f = eval("m."+function)
6068 lgerr.log("Error in Handler:" + str(sys.exc_info()[0]) + " " + str(sys.exc_info()[1]))
6069 traceback.print_tb(sys.exc_info()[2],None,lgerr)
6070 raise "No such function "+function+" in file "+self.file.filename
6072 def addPattern(self, pattern):
6073 p = WebPattern(self,pattern)
6074 desc = "pattern %s, file %s, function %s" % (pattern,self.file.filename,self.function)
6075 desc2 = "file %s, function %s" % (self.file.filename,self.function)
6076 self.file.context.pattern_to_function[p.getPattern()] = (self.f,desc)
6077 self.file.context.id_to_function["/"+self.function] = (self.f,desc2)
6081 def __init__(self, handler, pattern):
6082 self.handler = handler
6083 self.pattern = pattern
6084 if not pattern.endswith('$'):
6085 pattern = pattern + "$"
6086 self.compiled = re.compile(pattern)
6087 def getPattern(self):
6088 return self.compiled
6089 def getPatternString(self):
6092 def read_ini_file(filename):
6093 global GLOBAL_TEMP_DIR,GLOBAL_ROOT_DIR,number_of_threads,multithreading_enabled,contexts
6095 fi = open(filename, "rb")
6099 GLOBAL_ROOT_DIR = '/'
6100 for line in fi.readlines():
6102 hashpos = line.find("#")
6104 line = line[0:hashpos]
6108 continue #skip empty line
6110 equals = line.find(":")
6113 key = line[0:equals].strip()
6114 value = line[equals+1:].strip()
6115 if key == "tempdir":
6116 GLOBAL_TEMP_DIR = qualify_path(value)
6117 elif key == "threads":
6118 number_of_threads = int(value)
6119 multithreading_enabled = 1
6121 GLOBAL_ROOT_DIR = qualify_path(value)
6122 sys.path += [GLOBAL_ROOT_DIR]
6123 elif key == "filestore":
6124 if len(value) and value[0] != '/':
6126 filestore = FileStore(value)
6127 contexts += [filestore]
6129 elif key == "context":
6130 if len(value) and value[0] != '/':
6133 context = WebContext(contextname)
6134 contexts += [context]
6136 elif key == "startupfile":
6137 if context is not None:
6138 context.setStartupFile(value)
6140 raise "Error: startupfile must be below a context"
6142 if value.startswith('/'):
6145 context.setRoot(value)
6147 filestore.addRoot(value)
6150 context.addFile(filename)
6151 elif key == "ftphandler":
6152 file.addFTPHandler(value)
6153 elif key == "handler":
6155 file.addHandler(function)
6156 elif key == "macroresolver":
6157 file.addMacroResolver(value)
6158 elif key == "translator":
6159 file.addTranslator(value)
6160 elif key == "pattern":
6161 handler.addPattern(value)
6163 raise "Syntax error in line "+str(lineno)+" of file "+filename+":\n"+line
6166 def headers_to_map(mylist):
6174 key = h[0:i].lower()
6176 if len(value)>0 and value[0] == ' ':
6178 headers[key] = value
6180 if len(h.strip())>0:
6181 lg.log("invalid header: "+str(h))
6185 def __init__(self,fieldname, parammap,filename,content_type):
6186 self.fieldname = fieldname
6187 self.parammap = parammap
6188 self.filename = filename
6189 self.content_type = content_type
6190 self.tempname = GLOBAL_TEMP_DIR+str(int(random.random()*999999))+os.path.splitext(filename)[1]
6192 self.fi = open(self.tempname, "wb")
6193 def adddata(self,data):
6194 self.filesize += len(data)
6198 # only append file to parameters if it contains some data
6199 if self.filename or self.filesize:
6200 self.parammap[self.fieldname] = self
6205 return "file %s (%s), %d bytes, content-type: %s" % (self.filename, self.tempname, self.filesize, self.content_type)
6208 def __init__(self,fieldname,parammap):
6209 self.fieldname = fieldname
6211 self.parammap = parammap
6212 def adddata(self,data):
6216 oldvalue = self.parammap[self.fieldname] + ";"
6219 self.parammap[self.fieldname] = oldvalue + self.data
6223 class simple_input_collector:
6224 def __init__ (self, handler, request, length):
6225 self.request = request
6226 self.length = length
6227 self.handler = handler
6228 request.channel.set_terminator(length)
6231 def collect_incoming_data (self, data):
6234 def found_terminator(self):
6235 self.request.channel.set_terminator('\r\n\r\n')
6236 self.request.collector = None
6237 d=self.data;del self.data
6238 r=self.request;del self.request
6243 key,value = e.split('=')
6244 key = urllib.unquote_plus(key)
6246 oldvalue = parameters[key]+";"
6249 parameters[key] = oldvalue + urllib.unquote_plus(value)
6251 if len(e.strip())>0:
6252 lg.log("Unknown parameter: "+e)
6253 self.handler.continue_request(r,parameters)
6255 class upload_input_collector:
6256 def __init__ (self, handler, request, length, boundary):
6257 self.request = request
6258 self.length = length
6259 self.handler = handler
6260 self.boundary = boundary
6261 request.channel.set_terminator(length)
6264 self.start_marker = "--"+boundary+"\r\n"
6265 self.end_marker = "--"+boundary+"--\r\n"
6266 self.prefix = "--"+boundary
6267 self.marker = "\r\n--"+boundary
6268 self.header_end_marker = "\r\n\r\n"
6269 self.current_file = None
6270 self.boundary = boundary
6272 self.parameters = {}
6275 def parse_semicolon_parameters(self,params):
6276 params = params.split("; ")
6280 key,value = a.split('=')
6281 if value.startswith('"') and value.endswith('"'):
6286 def startFile(self,headers):
6289 if self.file is not None:
6290 raise "Illegal state"
6291 if "content-disposition" in headers:
6292 cd = headers["content-disposition"]
6293 l = self.parse_semicolon_parameters(cd)
6295 fieldname = l["name"]
6297 filename = l["filename"]
6298 if "content-type" in headers:
6299 content_type = headers["content-type"]
6300 self.file = AthanaFile(fieldname,self.parameters,filename,content_type)
6301 self.files += [self.file]
6303 self.file = AthanaField(fieldname,self.parameters)
6305 def split_headers(self,string):
6306 return string.split("\r\n")
6308 def collect_incoming_data (self, newdata):
6309 self.pos += len(newdata)
6310 self.data += newdata
6312 while len(self.data)>0:
6313 if self.data.startswith(self.end_marker):
6314 self.data = self.data[len(self.end_marker):]
6315 if self.file is not None:
6319 elif self.data.startswith(self.start_marker):
6321 i = self.data.index(self.header_end_marker, len(self.start_marker))
6325 headerstr = self.data[len(self.start_marker):i+2]
6326 headers = headers_to_map(self.split_headers(headerstr))
6327 self.startFile(headers)
6328 self.data = self.data[i+len(self.header_end_marker):]
6330 return # wait for more data (inside headers)
6331 elif self.data.startswith(self.prefix):
6335 bindex = self.data.index(self.marker)
6336 self.file.adddata(self.data[0:bindex])
6339 self.data = self.data[bindex+2:] # cut to position after \r\n
6340 except ValueError: #not found
6341 if(len(self.data) <= len(self.marker)):
6342 return #wait for more data before we make a decision or pass through data
6344 self.file.adddata(self.data[0:-len(self.marker)])
6345 self.data = self.data[-len(self.marker):]
6347 def found_terminator(self):
6348 if len(self.data)>0:# and self.file is not None:
6349 if self.file is not None:
6352 raise "Unfinished/malformed multipart request"
6353 if self.file is not None:
6357 self.request.collector = None
6358 self.request.channel.set_terminator('\r\n\r\n')
6359 d=self.data;del self.data
6360 r=self.request;del self.request
6361 r.tempfiles = [f.tempname for f in self.files]
6362 self.handler.continue_request(r,self.parameters)
6364 class Session(dict):
6365 def __init__(self, id):
6368 self.lastuse = time.time()
6370 def exception_string():
6371 s = "Exception "+str(sys.exc_info()[0])
6372 info = sys.exc_info()[1]
6376 for l in traceback.extract_tb(sys.exc_info()[2]):
6377 s += " File \"%s\", line %d, in %s\n" % (l[0],l[1],l[2])
6381 BASENAME = re.compile("([^/]*/)*([^/.]*)(.py)?")
6382 MULTIPART = re.compile ('multipart/form-data.*boundary=([^ ]*)', re.IGNORECASE)
6383 SESSION_PATTERN = re.compile("^;[a-z0-9]{6}-[a-z0-9]{6}-[a-z0-9]{6}$")
6387 class AthanaHandler:
6391 self.queuelock = thread.allocate_lock()
6393 def match(self, request):
6394 path, params, query, fragment = request.split_uri()
6395 #lg.log("===== request:"+path+"=====")
6398 def handle_request (self, request):
6399 headers = headers_to_map(request.header)
6400 request.request_headers = headers
6402 size=headers.get("content-length",None)
6404 if size and size != '0':
6406 ctype=headers.get("content-type",None)
6407 b = MULTIPART.match(ctype)
6409 request.type = "MULTIPART"
6410 boundary = b.group(1)
6411 request.collector = upload_input_collector(self,request,size,boundary)
6413 request.type = "POST"
6414 request.collector = simple_input_collector(self,request,size)
6416 request.type = "GET"
6417 self.continue_request(request, {})
6419 def create_session_id(self):
6420 pid = abs((str(random.random())).__hash__())
6421 now = abs((str(time.time())).__hash__())
6422 rand = abs((str(random.random())).__hash__())
6423 x = "abcdefghijklmnopqrstuvwxyz0123456789"
6425 for a in range(0,6):
6429 for a in range(0,6):
6433 for a in range(0,6):
6434 result += x[rand%36]
6438 def continue_request(self, request, parameters):
6440 path, params, query, fragment = request.split_uri()
6442 ip = request.request_headers.get("x-forwarded-for",None)
6444 try: ip = request.channel.addr[0]
6447 request.channel.addr = (ip,request.channel.addr[1])
6451 if query is not None:
6454 query = query.split('&')
6456 key,value = e.split('=')
6457 key = urllib.unquote_plus(key)
6459 oldvalue = parameters[key]+";"
6462 parameters[key] = oldvalue + urllib.unquote_plus(value) #_plus?
6465 if "cookie" in request.request_headers:
6466 cookiestr = request.request_headers["cookie"]
6467 if cookiestr.rfind(";") == len(cookiestr)-1:
6468 cookiestr = cookiestr[:-1]
6469 items = cookiestr.split(';')
6471 key,value = a.strip().split('=')
6472 cookies[key] = value
6474 request.Cookies = cookies
6477 if params is not None and SESSION_PATTERN.match(params):
6479 if sessionid[0] == ';':
6480 sessionid = sessionid[1:]
6481 elif use_cookies and "PSESSION" in cookies:
6482 sessionid = cookies["PSESSION"]
6484 if sessionid is not None:
6485 if sessionid in self.sessions:
6486 session = self.sessions[sessionid]
6489 session = Session(sessionid)
6490 self.sessions[sessionid] = session
6492 sessionid = self.create_session_id()
6493 session = Session(sessionid)
6494 self.sessions[sessionid] = session
6497 request['Connection'] = 'close';
6498 request['Content-Type'] = 'text/html; encoding=utf-8; charset=utf-8';
6504 #lg.debug("Compare context "+c.name+" with request "+path)
6505 if path.startswith(c.name) and len(c.name)>maxlen:
6507 maxlen = len(context.name)
6512 #print "Request ",'"'+path+'"',"maps to context",context.name
6514 path = path[len(context.name):]
6515 if len(path)==0 or path[0] != '/':
6518 request.session = session
6519 request.sessionid = sessionid
6520 request.context = context
6522 request.fullpath = fullpath
6523 request.paramstring = params
6524 request.query = query
6525 request.fragment = fragment
6526 request.params = parameters
6527 request.request = request
6529 request.uri = request.uri.replace(context.name, "/")
6530 request._split_uri = None
6533 request.setCookie('PSESSION', sessionid, time.time()+3600*2)
6535 request.channel.current_request = None
6537 function = context.match(path)
6539 if function is not None:
6540 if not multithreading_enabled:
6541 self.callhandler(function, request)
6543 self.queuelock.acquire()
6544 self.queue += [(function,request)]
6545 self.queuelock.release()
6548 lg.log("Request %s matches no pattern (context: %s)" % (request.path,context.name))
6549 return request.error(404, "File %s not found" % request.path)
6551 def callhandler(self, function, req):
6552 request = req.request
6555 status = function(req)
6557 lgerr.log("Error in page :" + str(sys.exc_info()[0]) + " " + str(sys.exc_info()[1]))
6558 traceback.print_tb(sys.exc_info()[2],None,lgerr)
6559 s = "<pre>"+exception_string()+"</pre>"
6560 return request.error(500,s)
6562 def worker_thread(server):
6564 server.queuelock.acquire()
6565 if len(server.queue) == 0:
6566 server.queuelock.release()
6569 function,req = server.queue.pop()
6570 server.queuelock.release()
6572 server.callhandler(function,req)
6574 lgerr.log("Error while processing request:" + str(sys.exc_info()[0]) + " " + str(sys.exc_info()[1]))
6575 traceback.print_tb(sys.exc_info()[2],None,lgerr)
6580 class virtual_authorizer:
6581 def __init__ (self):
6583 def authorize (self, channel, username, password):
6584 channel.persona = -1, -1
6585 channel.read_only = 1
6586 #return 1, 'Ok.', fs()
6587 return 1, 'Ok.', os_filesystem("/home/kramm")
6592 class logging_logger:
6593 def __init__(self,name="athana"):
6594 self.logger = logging.getLogger(name)
6595 def log (self, message):
6596 self.logger.info(message.rstrip())
6597 def debug (self, message):
6598 self.logger.debug(message.rstrip())
6599 def write (self, message):
6600 self.logger.info(message.rstrip())
6601 def error (self, message):
6602 self.logger.error(message.rstrip())
6604 lg = logging_logger()
6605 lgerr = logging_logger("errors")
6607 class zip_filesystem:
6608 def __init__(self, filename):
6609 self.filename = filename
6612 self.z = zipfile.ZipFile(filename)
6613 self.lock = thread.allocate_lock()
6614 for f in self.z.filelist:
6615 self.m['/' + f.filename] = f
6617 def current_directory(self):
6620 def isfile(self, path):
6621 if len(path) and path[-1]=='/':
6623 return (self.wd + path) in self.m
6625 def isdir (self, path):
6626 if not (len(path) and path[-1]=='/'):
6628 return path in self.m
6630 def cwd (self, path):
6631 path = join_paths(self.wd, path)
6632 if not self.isdir (path):
6640 i = self.wd[:-1].rindex('/')
6641 self.wd = self.wd[0:i+1]
6646 def listdir (self, path, long=0):
6647 raise "Not implemented"
6649 # TODO: implement a cache w/timeout for stat()
6650 def stat (self, path):
6651 fullpath = join_paths(self.wd, path)
6652 if self.isfile(path):
6653 size = self.m[fullpath].file_size
6654 return (33188, 77396L, 10L, 1, 1000, 1000, size, 0,0,0)
6655 elif self.isdir(path):
6656 return (16895, 117481L, 10L, 20, 1000, 1000, 4096L, 0,0,0)
6658 raise "No such file or directory "+path
6660 def open (self, path, mode):
6662 def __init__(self, content):
6663 self.content = content
6665 self.len = len(content)
6666 def read(self,l=None):
6668 l = self.len - self.pos
6669 if self.len < self.pos + l:
6670 l = self.len - self.pos
6671 s = self.content[self.pos : self.pos + l]
6680 data = self.z.read(path)
6685 def unlink (self, path):
6686 raise "Not implemented"
6687 def mkdir (self, path):
6688 raise "Not implemented"
6689 def rmdir (self, path):
6690 raise "Not implemented"
6692 def longify (self, (path, stat_info)):
6693 return unix_longify (path, stat_info)
6695 def __repr__ (self):
6696 return '<zipfile fs root:%s wd:%s>' % (self.filename, self.wd)
6700 global GLOBAL_ROOT_DIR
6701 GLOBAL_ROOT_DIR = qualify_path(base)
6703 def setTempDir(tempdir):
6704 global GLOBAL_TEMP_DIR
6705 GLOBAL_TEMP_DIR = qualify_path(tempdir)
6707 def addMacroResolver(m):
6708 global macroresolvers
6709 macroresolvers += [m]
6711 def addTranslator(m):
6715 def addFTPHandler(m):
6719 def addContext(webpath, localpath):
6721 c = WebContext(webpath, localpath)
6726 global contexts,translators,ftphandlers,macroresolvers,global_modules
6730 macroresolvers[:] = []
6731 global_modules.clear()
6732 _purge_all_modules()
6734 def addFileStore(webpath, localpaths):
6736 if len(webpath) and webpath[0] != '/':
6737 webpath = "/" + webpath
6738 c = FileStore(webpath, localpaths)
6742 def setThreads(number):
6743 global number_of_threads
6744 global multithreading_enabled
6746 multithreading_enabled=1
6747 number_of_threads=number
6749 multithreading_enabled=0
6754 ph = AthanaHandler()
6755 hs = http_server ('', port, logger_object = lg)
6756 hs.install_handler (ph)
6758 if len(ftphandlers) > 0:
6759 ftp = ftp_server (virtual_authorizer(), port=8021, logger_object=lg)
6761 if multithreading_enabled:
6763 for i in range(number_of_threads):
6764 threadlist += [thread.start_new_thread(worker_thread, (ph,))]
6768 asyncore.loop(timeout=0.01)
6769 except select.error:
6775 * temp directory in .cfg file
6778 def setTempDir(path):
6779 global GLOBAL_TEMP_DIR
6780 GLOBAL_TEMP_DIR = path
6783 global verbose,port,init_file,log_file,temp_path,multithreading_enabled,number_of_threads,GLOBAL_TEMP_DIR,contexts,lg,lgerr
6784 os.putenv('ATHANA_VERSION',ATHANA_VERSION)
6786 from optparse import OptionParser
6788 parser = OptionParser()
6790 parser.add_option("-v", "--verbose", dest="verbose", help="Be more verbose", action="store_true")
6791 parser.add_option("-q", "--quiet", dest="quiet", help="Be quiet", action="store_true")
6792 parser.add_option("-d", "--debug", dest="debug", help="Turn on debugging", action="store_true")
6793 parser.add_option("-p", "--port", dest="port", help="Set the port number", action="store",type="string")
6794 parser.add_option("-i", "--init-file", dest="init", help="Set the init file to use",action="store",type="string")
6795 parser.add_option("-l", "--log-file", dest="log", help="Set the logging file to use",action="store",type="string")
6796 parser.add_option("-t", "--temp-path", dest="temp", help="Set the temporary directory (default: /tmp/)",action="store",type="string")
6797 parser.add_option("-m", "--multithread", dest="multithreading_enabled", help="Enable multithreading",action="store_true")
6798 parser.add_option("-n", "--number-of-threads", dest="threads", help="Number of threads",action="store",type="int")
6799 parser.add_option("-T", "--talfile", dest="talfile", help="execute TAL File",action="store",type="string")
6801 (options, args) = parser.parse_args()
6809 if options.verbose != None : verbose = 2
6810 if options.quiet != None : verbose = 0
6811 if options.debug != None : verbose = 3
6812 if options.port != None : port = int(options.port)
6813 if options.init != None : init_file = options.init
6814 if options.log != None : log_file = options.log
6815 if options.temp != None : GLOBAL_TEMP_DIR = options.temp
6816 if options.multithreading_enabled : multithreading_enabled = 1
6817 if options.threads != None : number_of_threads = options.threads
6820 print getTAL(options.talfile, {"mynone":None})
6824 contexts += read_ini_file(inifile)
6826 if logfile is not None:
6827 fi = open(logfile, "wb")
6828 lg = file_logger (fi)
6832 if multithreading_enabled:
6833 print "Starting Athana (%d threads)..." % number_of_threads
6835 print "Starting Athana..."
6836 print "Init-File:",init_file
6837 print "Log-File:",log_file
6838 print "Temp-Path:",GLOBAL_TEMP_DIR
6843 if __name__ == '__main__':
6845 athana.mainfunction()