1
2 """
3 General Utilities
4 (part of web.py)
5 """
6
7 __all__ = [
8 "Storage", "storage", "storify",
9 "Counter", "counter",
10 "iters",
11 "rstrips", "lstrips", "strips",
12 "safeunicode", "safestr", "utf8",
13 "TimeoutError", "timelimit",
14 "Memoize", "memoize",
15 "re_compile", "re_subm",
16 "group", "uniq", "iterview",
17 "IterBetter", "iterbetter",
18 "safeiter", "safewrite",
19 "dictreverse", "dictfind", "dictfindall", "dictincr", "dictadd",
20 "requeue", "restack",
21 "listget", "intget", "datestr",
22 "numify", "denumify", "commify", "dateify",
23 "nthstr", "cond",
24 "CaptureStdout", "capturestdout", "Profile", "profile",
25 "tryall",
26 "ThreadedDict", "threadeddict",
27 "autoassign",
28 "to36",
29 "safemarkdown",
30 "sendmail"
31 ]
32
33 import re, sys, time, threading, itertools, traceback, os
34
35 try:
36 import subprocess
37 except ImportError:
38 subprocess = None
39
40 try: import datetime
41 except ImportError: pass
42
43 try: set
44 except NameError:
45 from sets import Set as set
46
48 """
49 A Storage object is like a dictionary except `obj.foo` can be used
50 in addition to `obj['foo']`.
51
52 >>> o = storage(a=1)
53 >>> o.a
54 1
55 >>> o['a']
56 1
57 >>> o.a = 2
58 >>> o['a']
59 2
60 >>> del o.a
61 >>> o.a
62 Traceback (most recent call last):
63 ...
64 AttributeError: 'a'
65
66 """
68 try:
69 return self[key]
70 except KeyError, k:
71 raise AttributeError, k
72
74 self[key] = value
75
77 try:
78 del self[key]
79 except KeyError, k:
80 raise AttributeError, k
81
83 return '<Storage ' + dict.__repr__(self) + '>'
84
85 storage = Storage
86
87 -def storify(mapping, *requireds, **defaults):
88 """
89 Creates a `storage` object from dictionary `mapping`, raising `KeyError` if
90 d doesn't have all of the keys in `requireds` and using the default
91 values for keys found in `defaults`.
92
93 For example, `storify({'a':1, 'c':3}, b=2, c=0)` will return the equivalent of
94 `storage({'a':1, 'b':2, 'c':3})`.
95
96 If a `storify` value is a list (e.g. multiple values in a form submission),
97 `storify` returns the last element of the list, unless the key appears in
98 `defaults` as a list. Thus:
99
100 >>> storify({'a':[1, 2]}).a
101 2
102 >>> storify({'a':[1, 2]}, a=[]).a
103 [1, 2]
104 >>> storify({'a':1}, a=[]).a
105 [1]
106 >>> storify({}, a=[]).a
107 []
108
109 Similarly, if the value has a `value` attribute, `storify will return _its_
110 value, unless the key appears in `defaults` as a dictionary.
111
112 >>> storify({'a':storage(value=1)}).a
113 1
114 >>> storify({'a':storage(value=1)}, a={}).a
115 <Storage {'value': 1}>
116 >>> storify({}, a={}).a
117 {}
118
119 Optionally, keyword parameter `_unicode` can be passed to convert all values to unicode.
120
121 >>> storify({'x': 'a'}, _unicode=True)
122 <Storage {'x': u'a'}>
123 >>> storify({'x': storage(value='a')}, x={}, _unicode=True)
124 <Storage {'x': <Storage {'value': 'a'}>}>
125 >>> storify({'x': storage(value='a')}, _unicode=True)
126 <Storage {'x': u'a'}>
127 """
128 _unicode = defaults.pop('_unicode', False)
129 def unicodify(s):
130 if _unicode and isinstance(s, str): return safeunicode(s)
131 else: return s
132
133 def getvalue(x):
134 if hasattr(x, 'file') and hasattr(x, 'value'):
135 return x.value
136 elif hasattr(x, 'value'):
137 return unicodify(x.value)
138 else:
139 return unicodify(x)
140
141 stor = Storage()
142 for key in requireds + tuple(mapping.keys()):
143 value = mapping[key]
144 if isinstance(value, list):
145 if isinstance(defaults.get(key), list):
146 value = [getvalue(x) for x in value]
147 else:
148 value = value[-1]
149 if not isinstance(defaults.get(key), dict):
150 value = getvalue(value)
151 if isinstance(defaults.get(key), list) and not isinstance(value, list):
152 value = [value]
153 setattr(stor, key, value)
154
155 for (key, value) in defaults.iteritems():
156 result = value
157 if hasattr(stor, key):
158 result = stor[key]
159 if value == () and not isinstance(result, tuple):
160 result = (result,)
161 setattr(stor, key, result)
162
163 return stor
164
166 """Keeps count of how many times something is added.
167
168 >>> c = counter()
169 >>> c.add('x')
170 >>> c.add('x')
171 >>> c.add('x')
172 >>> c.add('x')
173 >>> c.add('x')
174 >>> c.add('y')
175 >>> c
176 <Counter {'y': 1, 'x': 5}>
177 >>> c.most()
178 ['x']
179 """
181 self.setdefault(n, 0)
182 self[n] += 1
183
185 """Returns the keys with maximum count."""
186 m = max(self.itervalues())
187 return [k for k, v in self.iteritems() if v == m]
188
190 """Returns the keys with mininum count."""
191 m = min(self.itervalues())
192 return [k for k, v in self.iteritems() if v == m]
193
195 """Returns what percentage a certain key is of all entries.
196
197 >>> c = counter()
198 >>> c.add('x')
199 >>> c.add('x')
200 >>> c.add('x')
201 >>> c.add('y')
202 >>> c.percent('x')
203 0.75
204 >>> c.percent('y')
205 0.25
206 """
207 return float(self[key])/sum(self.values())
208
210 """Returns keys sorted by value.
211
212 >>> c = counter()
213 >>> c.add('x')
214 >>> c.add('x')
215 >>> c.add('y')
216 >>> c.sorted_keys()
217 ['x', 'y']
218 """
219 return sorted(self.keys(), key=lambda k: self[k], reverse=True)
220
222 """Returns values sorted by value.
223
224 >>> c = counter()
225 >>> c.add('x')
226 >>> c.add('x')
227 >>> c.add('y')
228 >>> c.sorted_values()
229 [2, 1]
230 """
231 return [self[k] for k in self.sorted_keys()]
232
234 """Returns items sorted by value.
235
236 >>> c = counter()
237 >>> c.add('x')
238 >>> c.add('x')
239 >>> c.add('y')
240 >>> c.sorted_items()
241 [('x', 2), ('y', 1)]
242 """
243 return [(k, self[k]) for k in self.sorted_keys()]
244
246 return '<Counter ' + dict.__repr__(self) + '>'
247
248 counter = Counter
249
250 iters = [list, tuple]
251 import __builtin__
252 if hasattr(__builtin__, 'set'):
253 iters.append(set)
254 if hasattr(__builtin__, 'frozenset'):
255 iters.append(set)
256 if sys.version_info < (2,6):
257 try:
258 from sets import Set
259 iters.append(Set)
260 except ImportError:
261 pass
262
264 iters = _hack(iters)
265 iters.__doc__ = """
266 A list of iterable items (like lists, but not strings). Includes whichever
267 of lists, tuples, sets, and Sets are available in this version of Python.
268 """
269
270 -def _strips(direction, text, remove):
271 if direction == 'l':
272 if text.startswith(remove):
273 return text[len(remove):]
274 elif direction == 'r':
275 if text.endswith(remove):
276 return text[:-len(remove)]
277 else:
278 raise ValueError, "Direction needs to be r or l."
279 return text
280
282 """
283 removes the string `remove` from the right of `text`
284
285 >>> rstrips("foobar", "bar")
286 'foo'
287
288 """
289 return _strips('r', text, remove)
290
292 """
293 removes the string `remove` from the left of `text`
294
295 >>> lstrips("foobar", "foo")
296 'bar'
297
298 """
299 return _strips('l', text, remove)
300
302 """
303 removes the string `remove` from the both sides of `text`
304
305 >>> strips("foobarfoo", "foo")
306 'bar'
307
308 """
309 return rstrips(lstrips(text, remove), remove)
310
312 r"""
313 Converts any given object to unicode string.
314
315 >>> safeunicode('hello')
316 u'hello'
317 >>> safeunicode(2)
318 u'2'
319 >>> safeunicode('\xe1\x88\xb4')
320 u'\u1234'
321 """
322 t = type(obj)
323 if t is unicode:
324 return obj
325 elif t is str:
326 return obj.decode(encoding)
327 elif t in [int, float, bool]:
328 return unicode(obj)
329 else:
330 if hasattr(obj, '__unicode__'):
331 return unicode(obj)
332 else:
333 return str(obj).decode(encoding)
334
335 -def safestr(obj, encoding='utf-8'):
336 r"""
337 Converts any given object to utf-8 encoded string.
338
339 >>> safestr('hello')
340 'hello'
341 >>> safestr(u'\u1234')
342 '\xe1\x88\xb4'
343 >>> safestr(2)
344 '2'
345 """
346 if isinstance(obj, unicode):
347 return obj.encode('utf-8')
348 elif isinstance(obj, str):
349 return obj
350 elif hasattr(obj, 'next') and hasattr(obj, '__iter__'):
351 return itertools.imap(safestr, obj)
352 else:
353 return str(obj)
354
355
356 utf8 = safestr
357
360 """
361 A decorator to limit a function to `timeout` seconds, raising `TimeoutError`
362 if it takes longer.
363
364 >>> import time
365 >>> def meaningoflife():
366 ... time.sleep(.2)
367 ... return 42
368 >>>
369 >>> timelimit(.1)(meaningoflife)()
370 Traceback (most recent call last):
371 ...
372 TimeoutError: took too long
373 >>> timelimit(1)(meaningoflife)()
374 42
375
376 _Caveat:_ The function isn't stopped after `timeout` seconds but continues
377 executing in a separate thread. (There seems to be no way to kill a thread.)
378
379 inspired by <http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/473878>
380 """
381 def _1(function):
382 def _2(*args, **kw):
383 class Dispatch(threading.Thread):
384 def __init__(self):
385 threading.Thread.__init__(self)
386 self.result = None
387 self.error = None
388
389 self.setDaemon(True)
390 self.start()
391
392 def run(self):
393 try:
394 self.result = function(*args, **kw)
395 except:
396 self.error = sys.exc_info()
397
398 c = Dispatch()
399 c.join(timeout)
400 if c.isAlive():
401 raise TimeoutError, 'took too long'
402 if c.error:
403 raise c.error[0], c.error[1]
404 return c.result
405 return _2
406 return _1
407
409 """
410 'Memoizes' a function, caching its return values for each input.
411 If `expires` is specified, values are recalculated after `expires` seconds.
412 If `background` is specified, values are recalculated in a separate thread.
413
414 >>> calls = 0
415 >>> def howmanytimeshaveibeencalled():
416 ... global calls
417 ... calls += 1
418 ... return calls
419 >>> fastcalls = memoize(howmanytimeshaveibeencalled)
420 >>> howmanytimeshaveibeencalled()
421 1
422 >>> howmanytimeshaveibeencalled()
423 2
424 >>> fastcalls()
425 3
426 >>> fastcalls()
427 3
428 >>> import time
429 >>> fastcalls = memoize(howmanytimeshaveibeencalled, .1, background=False)
430 >>> fastcalls()
431 4
432 >>> fastcalls()
433 4
434 >>> time.sleep(.2)
435 >>> fastcalls()
436 5
437 >>> def slowfunc():
438 ... time.sleep(.1)
439 ... return howmanytimeshaveibeencalled()
440 >>> fastcalls = memoize(slowfunc, .2, background=True)
441 >>> fastcalls()
442 6
443 >>> timelimit(.05)(fastcalls)()
444 6
445 >>> time.sleep(.2)
446 >>> timelimit(.05)(fastcalls)()
447 6
448 >>> timelimit(.05)(fastcalls)()
449 6
450 >>> time.sleep(.2)
451 >>> timelimit(.05)(fastcalls)()
452 7
453 >>> fastcalls = memoize(slowfunc, None, background=True)
454 >>> threading.Thread(target=fastcalls).start()
455 >>> time.sleep(.01)
456 >>> fastcalls()
457 9
458 """
459 - def __init__(self, func, expires=None, background=True):
460 self.func = func
461 self.cache = {}
462 self.expires = expires
463 self.background = background
464 self.running = {}
465
467 key = (args, tuple(keywords.items()))
468 if not self.running.get(key):
469 self.running[key] = threading.Lock()
470 def update(block=False):
471 if self.running[key].acquire(block):
472 try:
473 self.cache[key] = (self.func(*args, **keywords), time.time())
474 finally:
475 self.running[key].release()
476
477 if key not in self.cache:
478 update(block=True)
479 elif self.expires and (time.time() - self.cache[key][1]) > self.expires:
480 if self.background:
481 threading.Thread(target=update).start()
482 else:
483 update()
484 return self.cache[key][0]
485
486 memoize = Memoize
487
488 re_compile = memoize(re.compile)
489 re_compile.__doc__ = """
490 A memoized version of re.compile.
491 """
492
497 self.match = match
498 return ''
499
501 """
502 Like re.sub, but returns the replacement _and_ the match object.
503
504 >>> t, m = re_subm('g(oo+)fball', r'f\\1lish', 'goooooofball')
505 >>> t
506 'foooooolish'
507 >>> m.groups()
508 ('oooooo',)
509 """
510 compiled_pat = re_compile(pat)
511 proxy = _re_subm_proxy()
512 compiled_pat.sub(proxy.__call__, string)
513 return compiled_pat.sub(repl, string), proxy.match
514
516 """
517 Returns an iterator over a series of lists of length size from iterable.
518
519 >>> list(group([1,2,3,4], 2))
520 [[1, 2], [3, 4]]
521 >>> list(group([1,2,3,4,5], 2))
522 [[1, 2], [3, 4], [5]]
523 """
524 def take(seq, n):
525 for i in xrange(n):
526 yield seq.next()
527
528 if not hasattr(seq, 'next'):
529 seq = iter(seq)
530 while True:
531 x = list(take(seq, size))
532 if x:
533 yield x
534 else:
535 break
536
538 """
539 Removes duplicate elements from a list.
540
541 >>> uniq([1,2,3,1,4,5,6])
542 [1, 2, 3, 4, 5, 6]
543 """
544 seen = set()
545 result = []
546 for item in seq:
547 if item in seen: continue
548 seen.add(item)
549 result.append(item)
550 return result
551
553 """
554 Takes an iterable `x` and returns an iterator over it
555 which prints its progress to stderr as it iterates through.
556 """
557 WIDTH = 70
558
559 def plainformat(n, lenx):
560 return '%5.1f%% (%*d/%d)' % ((float(n)/lenx)*100, len(str(lenx)), n, lenx)
561
562 def bars(size, n, lenx):
563 val = int((float(n)*size)/lenx + 0.5)
564 if size - val:
565 spacing = ">" + (" "*(size-val))[1:]
566 else:
567 spacing = ""
568 return "[%s%s]" % ("="*val, spacing)
569
570 def eta(elapsed, n, lenx):
571 if n == 0:
572 return '--:--:--'
573 if n == lenx:
574 secs = int(elapsed)
575 else:
576 secs = int((elapsed/n) * (lenx-n))
577 mins, secs = divmod(secs, 60)
578 hrs, mins = divmod(mins, 60)
579
580 return '%02d:%02d:%02d' % (hrs, mins, secs)
581
582 def format(starttime, n, lenx):
583 out = plainformat(n, lenx) + ' '
584 if n == lenx:
585 end = ' '
586 else:
587 end = ' ETA '
588 end += eta(time.time() - starttime, n, lenx)
589 out += bars(WIDTH - len(out) - len(end), n, lenx)
590 out += end
591 return out
592
593 starttime = time.time()
594 lenx = len(x)
595 for n, y in enumerate(x):
596 sys.stderr.write('\r' + format(starttime, n, lenx))
597 yield y
598 sys.stderr.write('\r' + format(starttime, n+1, lenx) + '\n')
599
601 """
602 Returns an object that can be used as an iterator
603 but can also be used via __getitem__ (although it
604 cannot go backwards -- that is, you cannot request
605 `iterbetter[0]` after requesting `iterbetter[1]`).
606
607 >>> import itertools
608 >>> c = iterbetter(itertools.count())
609 >>> c[1]
610 1
611 >>> c[5]
612 5
613 >>> c[3]
614 Traceback (most recent call last):
615 ...
616 IndexError: already passed 3
617
618 For boolean test, IterBetter peeps at first value in the itertor without effecting the iteration.
619
620 >>> c = iterbetter(iter(range(5)))
621 >>> bool(c)
622 True
623 >>> list(c)
624 [0, 1, 2, 3, 4]
625 >>> c = iterbetter(iter([]))
626 >>> bool(c)
627 False
628 >>> list(c)
629 []
630 """
632 self.i, self.c = iterator, 0
633
635 if hasattr(self, "_head"):
636 yield self._head
637
638 while 1:
639 yield self.i.next()
640 self.c += 1
641
643
644 if i < self.c:
645 raise IndexError, "already passed "+str(i)
646 try:
647 while i > self.c:
648 self.i.next()
649 self.c += 1
650
651 self.c += 1
652 return self.i.next()
653 except StopIteration:
654 raise IndexError, str(i)
655
657 if hasattr(self, "__len__"):
658 return len(self) != 0
659 elif hasattr(self, "_head"):
660 return True
661 else:
662 try:
663 self._head = self.i.next()
664 except StopIteration:
665 return False
666 else:
667 return True
668
669 iterbetter = IterBetter
670
671 -def safeiter(it, cleanup=None, ignore_errors=True):
672 """Makes an iterator safe by ignoring the exceptions occured during the iteration.
673 """
674 def next():
675 while True:
676 try:
677 return it.next()
678 except StopIteration:
679 raise
680 except:
681 traceback.print_exc()
682
683 it = iter(it)
684 while True:
685 yield next()
686
688 """Writes the content to a temp file and then moves the temp file to
689 given filename to avoid overwriting the existing file in case of errors.
690 """
691 f = file(filename + '.tmp', 'w')
692 f.write(content)
693 f.close()
694 os.rename(f.name, path)
695
697 """
698 Returns a new dictionary with keys and values swapped.
699
700 >>> dictreverse({1: 2, 3: 4})
701 {2: 1, 4: 3}
702 """
703 return dict([(value, key) for (key, value) in mapping.iteritems()])
704
706 """
707 Returns a key whose value in `dictionary` is `element`
708 or, if none exists, None.
709
710 >>> d = {1:2, 3:4}
711 >>> dictfind(d, 4)
712 3
713 >>> dictfind(d, 5)
714 """
715 for (key, value) in dictionary.iteritems():
716 if element is value:
717 return key
718
720 """
721 Returns the keys whose values in `dictionary` are `element`
722 or, if none exists, [].
723
724 >>> d = {1:4, 3:4}
725 >>> dictfindall(d, 4)
726 [1, 3]
727 >>> dictfindall(d, 5)
728 []
729 """
730 res = []
731 for (key, value) in dictionary.iteritems():
732 if element is value:
733 res.append(key)
734 return res
735
737 """
738 Increments `element` in `dictionary`,
739 setting it to one if it doesn't exist.
740
741 >>> d = {1:2, 3:4}
742 >>> dictincr(d, 1)
743 3
744 >>> d[1]
745 3
746 >>> dictincr(d, 5)
747 1
748 >>> d[5]
749 1
750 """
751 dictionary.setdefault(element, 0)
752 dictionary[element] += 1
753 return dictionary[element]
754
756 """
757 Returns a dictionary consisting of the keys in the argument dictionaries.
758 If they share a key, the value from the last argument is used.
759
760 >>> dictadd({1: 0, 2: 0}, {2: 1, 3: 1})
761 {1: 0, 2: 1, 3: 1}
762 """
763 result = {}
764 for dct in dicts:
765 result.update(dct)
766 return result
767
769 """Returns the element at index after moving it to the beginning of the queue.
770
771 >>> x = [1, 2, 3, 4]
772 >>> requeue(x)
773 4
774 >>> x
775 [4, 1, 2, 3]
776 """
777 x = queue.pop(index)
778 queue.insert(0, x)
779 return x
780
782 """Returns the element at index after moving it to the top of stack.
783
784 >>> x = [1, 2, 3, 4]
785 >>> restack(x)
786 1
787 >>> x
788 [2, 3, 4, 1]
789 """
790 x = stack.pop(index)
791 stack.append(x)
792 return x
793
794 -def listget(lst, ind, default=None):
795 """
796 Returns `lst[ind]` if it exists, `default` otherwise.
797
798 >>> listget(['a'], 0)
799 'a'
800 >>> listget(['a'], 1)
801 >>> listget(['a'], 1, 'b')
802 'b'
803 """
804 if len(lst)-1 < ind:
805 return default
806 return lst[ind]
807
808 -def intget(integer, default=None):
809 """
810 Returns `integer` as an int or `default` if it can't.
811
812 >>> intget('3')
813 3
814 >>> intget('3a')
815 >>> intget('3a', 0)
816 0
817 """
818 try:
819 return int(integer)
820 except (TypeError, ValueError):
821 return default
822
824 """
825 Converts a (UTC) datetime object to a nice string representation.
826
827 >>> from datetime import datetime, timedelta
828 >>> d = datetime(1970, 5, 1)
829 >>> datestr(d, now=d)
830 '0 microseconds ago'
831 >>> for t, v in {
832 ... timedelta(microseconds=1): '1 microsecond ago',
833 ... timedelta(microseconds=2): '2 microseconds ago',
834 ... -timedelta(microseconds=1): '1 microsecond from now',
835 ... -timedelta(microseconds=2): '2 microseconds from now',
836 ... timedelta(microseconds=2000): '2 milliseconds ago',
837 ... timedelta(seconds=2): '2 seconds ago',
838 ... timedelta(seconds=2*60): '2 minutes ago',
839 ... timedelta(seconds=2*60*60): '2 hours ago',
840 ... timedelta(days=2): '2 days ago',
841 ... }.iteritems():
842 ... assert datestr(d, now=d+t) == v
843 >>> datestr(datetime(1970, 1, 1), now=d)
844 'January 1'
845 >>> datestr(datetime(1969, 1, 1), now=d)
846 'January 1, 1969'
847 >>> datestr(datetime(1970, 6, 1), now=d)
848 'June 1, 1970'
849 >>> datestr(None)
850 ''
851 """
852 def agohence(n, what, divisor=None):
853 if divisor: n = n // divisor
854
855 out = str(abs(n)) + ' ' + what
856 if abs(n) != 1: out += 's'
857 out += ' '
858 if n < 0:
859 out += 'from now'
860 else:
861 out += 'ago'
862 return out
863
864 oneday = 24 * 60 * 60
865
866 if not then: return ""
867 if not now: now = datetime.datetime.utcnow()
868 if type(now).__name__ == "DateTime":
869 now = datetime.datetime.fromtimestamp(now)
870 if type(then).__name__ == "DateTime":
871 then = datetime.datetime.fromtimestamp(then)
872 elif type(then).__name__ == "date":
873 then = datetime.datetime(then.year, then.month, then.day)
874
875 delta = now - then
876 deltaseconds = int(delta.days * oneday + delta.seconds + delta.microseconds * 1e-06)
877 deltadays = abs(deltaseconds) // oneday
878 if deltaseconds < 0: deltadays *= -1
879
880 if deltadays:
881 if abs(deltadays) < 4:
882 return agohence(deltadays, 'day')
883
884 out = then.strftime('%B %e')
885 if then.year != now.year or deltadays < 0:
886 out += ', %s' % then.year
887 return out
888
889 if int(deltaseconds):
890 if abs(deltaseconds) > (60 * 60):
891 return agohence(deltaseconds, 'hour', 60 * 60)
892 elif abs(deltaseconds) > 60:
893 return agohence(deltaseconds, 'minute', 60)
894 else:
895 return agohence(deltaseconds, 'second')
896
897 deltamicroseconds = delta.microseconds
898 if delta.days: deltamicroseconds = int(delta.microseconds - 1e6)
899 if abs(deltamicroseconds) > 1000:
900 return agohence(deltamicroseconds, 'millisecond', 1000)
901
902 return agohence(deltamicroseconds, 'microsecond')
903
905 """
906 Removes all non-digit characters from `string`.
907
908 >>> numify('800-555-1212')
909 '8005551212'
910 >>> numify('800.555.1212')
911 '8005551212'
912
913 """
914 return ''.join([c for c in str(string) if c.isdigit()])
915
917 """
918 Formats `string` according to `pattern`, where the letter X gets replaced
919 by characters from `string`.
920
921 >>> denumify("8005551212", "(XXX) XXX-XXXX")
922 '(800) 555-1212'
923
924 """
925 out = []
926 for c in pattern:
927 if c == "X":
928 out.append(string[0])
929 string = string[1:]
930 else:
931 out.append(c)
932 return ''.join(out)
933
935 """
936 Add commas to an integer `n`.
937
938 >>> commify(1)
939 '1'
940 >>> commify(123)
941 '123'
942 >>> commify(1234)
943 '1,234'
944 >>> commify(1234567890)
945 '1,234,567,890'
946 >>> commify(123.0)
947 '123.0'
948 >>> commify(1234.5)
949 '1,234.5'
950 >>> commify(1234.56789)
951 '1,234.56789'
952 >>> commify('%.2f' % 1234.5)
953 '1,234.50'
954 >>> commify(None)
955 >>>
956
957 """
958 if n is None: return None
959 n = str(n)
960 if '.' in n:
961 dollars, cents = n.split('.')
962 else:
963 dollars, cents = n, None
964
965 r = []
966 for i, c in enumerate(str(dollars)[::-1]):
967 if i and (not (i % 3)):
968 r.insert(0, ',')
969 r.insert(0, c)
970 out = ''.join(r)
971 if cents:
972 out += '.' + cents
973 return out
974
976 """
977 Formats a numified `datestring` properly.
978 """
979 return denumify(datestring, "XXXX-XX-XX XX:XX:XX")
980
981
983 """
984 Formats an ordinal.
985 Doesn't handle negative numbers.
986
987 >>> nthstr(1)
988 '1st'
989 >>> nthstr(0)
990 '0th'
991 >>> [nthstr(x) for x in [2, 3, 4, 5, 10, 11, 12, 13, 14, 15]]
992 ['2nd', '3rd', '4th', '5th', '10th', '11th', '12th', '13th', '14th', '15th']
993 >>> [nthstr(x) for x in [91, 92, 93, 94, 99, 100, 101, 102]]
994 ['91st', '92nd', '93rd', '94th', '99th', '100th', '101st', '102nd']
995 >>> [nthstr(x) for x in [111, 112, 113, 114, 115]]
996 ['111th', '112th', '113th', '114th', '115th']
997
998 """
999
1000 assert n >= 0
1001 if n % 100 in [11, 12, 13]: return '%sth' % n
1002 return {1: '%sst', 2: '%snd', 3: '%srd'}.get(n % 10, '%sth') % n
1003
1004 -def cond(predicate, consequence, alternative=None):
1005 """
1006 Function replacement for if-else to use in expressions.
1007
1008 >>> x = 2
1009 >>> cond(x % 2 == 0, "even", "odd")
1010 'even'
1011 >>> cond(x % 2 == 0, "even", "odd") + '_row'
1012 'even_row'
1013 """
1014 if predicate:
1015 return consequence
1016 else:
1017 return alternative
1018
1020 """
1021 Captures everything `func` prints to stdout and returns it instead.
1022
1023 >>> def idiot():
1024 ... print "foo"
1025 >>> capturestdout(idiot)()
1026 'foo\\n'
1027
1028 **WARNING:** Not threadsafe!
1029 """
1031 self.func = func
1032 - def __call__(self, *args, **keywords):
1033 from cStringIO import StringIO
1034
1035 out = StringIO()
1036 oldstdout = sys.stdout
1037 sys.stdout = out
1038 try:
1039 self.func(*args, **keywords)
1040 finally:
1041 sys.stdout = oldstdout
1042 return out.getvalue()
1043
1044 capturestdout = CaptureStdout
1045
1047 """
1048 Profiles `func` and returns a tuple containing its output
1049 and a string with human-readable profiling information.
1050
1051 >>> import time
1052 >>> out, inf = profile(time.sleep)(.001)
1053 >>> out
1054 >>> inf[:10].strip()
1055 'took 0.0'
1056 """
1058 self.func = func
1060 import hotshot, hotshot.stats, os, tempfile
1061 f, filename = tempfile.mkstemp()
1062 os.close(f)
1063
1064 prof = hotshot.Profile(filename)
1065
1066 stime = time.time()
1067 result = prof.runcall(self.func, *args)
1068 stime = time.time() - stime
1069 prof.close()
1070
1071 import cStringIO
1072 out = cStringIO.StringIO()
1073 stats = hotshot.stats.load(filename)
1074 stats.stream = out
1075 stats.strip_dirs()
1076 stats.sort_stats('time', 'calls')
1077 stats.print_stats(40)
1078 stats.print_callers()
1079
1080 x = '\n\ntook '+ str(stime) + ' seconds\n'
1081 x += out.getvalue()
1082
1083
1084 try:
1085 os.remove(filename)
1086 except IOError:
1087 pass
1088
1089 return result, x
1090
1091 profile = Profile
1092
1093
1094 import traceback
1095
1096 if not hasattr(traceback, 'format_exc'):
1097 from cStringIO import StringIO
1102 traceback.format_exc = format_exc
1103
1104 -def tryall(context, prefix=None):
1105 """
1106 Tries a series of functions and prints their results.
1107 `context` is a dictionary mapping names to values;
1108 the value will only be tried if it's callable.
1109
1110 >>> tryall(dict(j=lambda: True))
1111 j: True
1112 ----------------------------------------
1113 results:
1114 True: 1
1115
1116 For example, you might have a file `test/stuff.py`
1117 with a series of functions testing various things in it.
1118 At the bottom, have a line:
1119
1120 if __name__ == "__main__": tryall(globals())
1121
1122 Then you can run `python test/stuff.py` and get the results of
1123 all the tests.
1124 """
1125 context = context.copy()
1126 results = {}
1127 for (key, value) in context.iteritems():
1128 if not hasattr(value, '__call__'):
1129 continue
1130 if prefix and not key.startswith(prefix):
1131 continue
1132 print key + ':',
1133 try:
1134 r = value()
1135 dictincr(results, r)
1136 print r
1137 except:
1138 print 'ERROR'
1139 dictincr(results, 'ERROR')
1140 print ' ' + '\n '.join(traceback.format_exc().split('\n'))
1141
1142 print '-'*40
1143 print 'results:'
1144 for (key, value) in results.iteritems():
1145 print ' '*2, str(key)+':', value
1146
1148 """
1149 Thread local storage.
1150
1151 >>> d = ThreadedDict()
1152 >>> d.x = 1
1153 >>> d.x
1154 1
1155 >>> import threading
1156 >>> def f(): d.x = 2
1157 ...
1158 >>> t = threading.Thread(target=f)
1159 >>> t.start()
1160 >>> t.join()
1161 >>> d.x
1162 1
1163 """
1165 return getattr(self._getd(), key)
1166
1168 return setattr(self._getd(), key, value)
1169
1171 return delattr(self._getd(), key)
1172
1174 return id(self)
1175
1177 t = threading.currentThread()
1178 if not hasattr(t, '_d'):
1179
1180 t._d = {}
1181
1182
1183
1184 if self not in t._d:
1185 t._d[self] = storage()
1186 return t._d[self]
1187
1188 threadeddict = ThreadedDict
1189
1191 """
1192 Automatically assigns local variables to `self`.
1193
1194 >>> self = storage()
1195 >>> autoassign(self, dict(a=1, b=2))
1196 >>> self
1197 <Storage {'a': 1, 'b': 2}>
1198
1199 Generally used in `__init__` methods, as in:
1200
1201 def __init__(self, foo, bar, baz=1): autoassign(self, locals())
1202 """
1203 for (key, value) in locals.iteritems():
1204 if key == 'self':
1205 continue
1206 setattr(self, key, value)
1207
1209 """
1210 Converts an integer to base 36 (a useful scheme for human-sayable IDs).
1211
1212 >>> to36(35)
1213 'z'
1214 >>> to36(119292)
1215 '2k1o'
1216 >>> int(to36(939387374), 36)
1217 939387374
1218 >>> to36(0)
1219 '0'
1220 >>> to36(-393)
1221 Traceback (most recent call last):
1222 ...
1223 ValueError: must supply a positive integer
1224
1225 """
1226 if q < 0: raise ValueError, "must supply a positive integer"
1227 letters = "0123456789abcdefghijklmnopqrstuvwxyz"
1228 converted = []
1229 while q != 0:
1230 q, r = divmod(q, 36)
1231 converted.insert(0, letters[r])
1232 return "".join(converted) or '0'
1233
1234
1235 r_url = re_compile('(?<!\()(http://(\S+))')
1237 """
1238 Converts text to HTML following the rules of Markdown, but blocking any
1239 outside HTML input, so that only the things supported by Markdown
1240 can be used. Also converts raw URLs to links.
1241
1242 (requires [markdown.py](http://webpy.org/markdown.py))
1243 """
1244 from markdown import markdown
1245 if text:
1246 text = text.replace('<', '<')
1247
1248 text = r_url.sub(r'<\1>', text)
1249 text = markdown(text)
1250 return text
1251
1252 -def sendmail(from_address, to_address, subject, message, headers=None, **kw):
1253 """
1254 Sends the email message `message` with mail and envelope headers
1255 for from `from_address_` to `to_address` with `subject`.
1256 Additional email headers can be specified with the dictionary
1257 `headers.
1258
1259 Optionally cc, bcc and attachments can be specified as keyword arguments.
1260 Attachments must be an iterable and each attachment can be either a
1261 filename or a file object or a dictionary with filename, content and
1262 optionally content_type keys.
1263
1264 If `web.config.smtp_server` is set, it will send the message
1265 to that SMTP server. Otherwise it will look for
1266 `/usr/sbin/sendmail`, the typical location for the sendmail-style
1267 binary. To use sendmail from a different path, set `web.config.sendmail_path`.
1268 """
1269 attachments = kw.pop("attachments", [])
1270 mail = _EmailMessage(from_address, to_address, subject, message, headers, **kw)
1271
1272 for a in attachments:
1273 if isinstance(a, dict):
1274 mail.attach(a['filename'], a['content'], a.get('content_type'))
1275 elif hasattr(a, 'read'):
1276 filename = os.path.basename(getattr(a, "name", ""))
1277 content_type = getattr(a, 'content_type', None)
1278 mail.attach(filename, a.read(), content_type)
1279 elif isinstance(a, basestring):
1280 f = open(a, 'rb')
1281 content = f.read()
1282 f.close()
1283 filename = os.path.basename(a)
1284 mail.attach(filename, content, None)
1285 else:
1286 raise ValueError, "Invalid attachment: %s" % repr(a)
1287
1288 mail.send()
1289
1291 - def __init__(self, from_address, to_address, subject, message, headers=None, **kw):
1292 def listify(x):
1293 if not isinstance(x, list):
1294 return [safestr(x)]
1295 else:
1296 return [safestr(a) for a in x]
1297
1298 subject = safestr(subject)
1299 message = safestr(message)
1300
1301 from_address = safestr(from_address)
1302 to_address = listify(to_address)
1303 cc = listify(kw.get('cc', []))
1304 bcc = listify(kw.get('bcc', []))
1305 recipients = to_address + cc + bcc
1306
1307 import email.Utils
1308 self.from_address = email.Utils.parseaddr(from_address)[1]
1309 self.recipients = [email.Utils.parseaddr(r)[1] for r in recipients]
1310
1311 self.headers = dictadd({
1312 'From': from_address,
1313 'To': ", ".join(to_address),
1314 'Subject': subject
1315 }, headers or {})
1316
1317 if cc:
1318 self.headers['Cc'] = ", ".join(cc)
1319
1320 self.message = self.new_message()
1321 self.message.add_header("Content-Transfer-Encoding", "7bit")
1322 self.message.add_header("Content-Disposition", "inline")
1323 self.message.add_header("MIME-Version", "1.0")
1324 self.message.set_payload(message, 'utf-8')
1325 self.multipart = False
1331 - def attach(self, filename, content, content_type=None):
1332 if not self.multipart:
1333 msg = self.new_message()
1334 msg.add_header("Content-Type", "multipart/mixed")
1335 msg.attach(self.message)
1336 self.message = msg
1337 self.multipart = True
1338
1339 import mimetypes
1340 try:
1341 from email import encoders
1342 except:
1343 from email import Encoders as encoders
1344
1345 content_type = content_type or mimetypes.guess_type(filename)[0] or "applcation/octet-stream"
1346
1347 msg = self.new_message()
1348 msg.set_payload(content)
1349 msg.add_header('Content-Type', content_type)
1350 msg.add_header('Content-Disposition', 'attachment', filename=filename)
1351
1352 if not content_type.startswith("text/"):
1353 encoders.encode_base64(msg)
1354
1355 self.message.attach(msg)
1356
1358 for k, v in self.headers.iteritems():
1359 if k.lower() == "content-type":
1360 self.message.set_type(v)
1361 else:
1362 self.message.add_header(k, v)
1363
1364 self.headers = {}
1365
1367 try:
1368 import webapi
1369 except ImportError:
1370 webapi = Storage(config=Storage())
1371
1372 self.prepare_message()
1373 message_text = self.message.as_string()
1374
1375 if webapi.config.get('smtp_server'):
1376 server = webapi.config.get('smtp_server')
1377 port = webapi.config.get('smtp_port', 0)
1378 username = webapi.config.get('smtp_username')
1379 password = webapi.config.get('smtp_password')
1380 debug_level = webapi.config.get('smtp_debuglevel', None)
1381 starttls = webapi.config.get('smtp_starttls', False)
1382
1383 import smtplib
1384 smtpserver = smtplib.SMTP(server, port)
1385
1386 if debug_level:
1387 smtpserver.set_debuglevel(debug_level)
1388
1389 if starttls:
1390 smtpserver.ehlo()
1391 smtpserver.starttls()
1392 smtpserver.ehlo()
1393
1394 if username and password:
1395 smtpserver.login(username, password)
1396
1397 smtpserver.sendmail(self.from_address, self.recipients, message_text)
1398 smtpserver.quit()
1399 else:
1400 sendmail = webapi.config.get('sendmail_path', '/usr/sbin/sendmail')
1401
1402 assert not self.from_address.startswith('-'), 'security'
1403 for r in self.recipients:
1404 assert not r.startswith('-'), 'security'
1405
1406 cmd = [sendmail, '-f', self.from_address] + self.recipients
1407
1408 if subprocess:
1409 p = subprocess.Popen(cmd, stdin=subprocess.PIPE)
1410 p.stdin.write(message_text)
1411 p.stdin.close()
1412 p.wait()
1413 else:
1414 i, o = os.popen2(cmd)
1415 i.write(message)
1416 i.close()
1417 o.close()
1418 del i, o
1419
1421 return "<EmailMessage>"
1422
1424 return self.message.as_string()
1425
1426 if __name__ == "__main__":
1427 import doctest
1428 doctest.testmod()
1429