web.web.utils

Package web :: Package web :: Module utils |
[hide private]
[frames] | no frames]

# Source Code for Module web.web.utils

       1  #!/usr/bin/env python
       2  """
       3  General Utilities
       4  (part of web.py)
       5  """
       6
       7  __all__ = [
       8    "Storage", "storage", "storify",
       9    "Counter", "counter",
      10    "iters",
      11    "rstrips", "lstrips", "strips",
      12    "safeunicode", "safestr", "utf8",
      13    "TimeoutError", "timelimit",
      14    "Memoize", "memoize",
      15    "re_compile", "re_subm",
      16    "group", "uniq", "iterview",
      17    "IterBetter", "iterbetter",
      18    "safeiter", "safewrite",
      19    "dictreverse", "dictfind", "dictfindall", "dictincr", "dictadd",
      20    "requeue", "restack",
      21    "listget", "intget", "datestr",
      22    "numify", "denumify", "commify", "dateify",
      23    "nthstr", "cond",
      24    "CaptureStdout", "capturestdout", "Profile", "profile",
      25    "tryall",
      26    "ThreadedDict", "threadeddict",
      27    "autoassign",
      28    "to36",
      29    "safemarkdown",
      30    "sendmail"
      31  ]
      32
      33  import re, sys, time, threading, itertools, traceback, os
      34
      35  try:
      36      import subprocess
      37  except ImportError:
      38      subprocess = None
      39
      40  try: import datetime
      41  except ImportError: pass
      42
      43  try: set
      44  except NameError:
      45      from sets import Set as set
      46
      47 -class Storage(dict):
      48      """
      49      A Storage object is like a dictionary except `obj.foo` can be used
      50      in addition to `obj['foo']`.
      51
      52          >>> o = storage(a=1)
      53          >>> o.a
      54          1
      55          >>> o['a']
      56          1
      57          >>> o.a = 2
      58          >>> o['a']
      59          2
      60          >>> del o.a
      61          >>> o.a
      62          Traceback (most recent call last):
      63              ...
      64          AttributeError: 'a'
      65
      66      """
      67 -    def __getattr__(self, key):
      68          try:
      69              return self[key]
      70          except KeyError, k:
      71              raise AttributeError, k
      72
      73 -    def __setattr__(self, key, value):
      74          self[key] = value
      75
      76 -    def __delattr__(self, key):
      77          try:
      78              del self[key]
      79          except KeyError, k:
      80              raise AttributeError, k
      81
      82 -    def __repr__(self):
      83          return '<Storage ' + dict.__repr__(self) + '>'
      84
      85  storage = Storage
      86
      87 -def storify(mapping, *requireds, **defaults):
      88      """
      89      Creates a `storage` object from dictionary `mapping`, raising `KeyError` if
      90      d doesn't have all of the keys in `requireds` and using the default
      91      values for keys found in `defaults`.
      92
      93      For example, `storify({'a':1, 'c':3}, b=2, c=0)` will return the equivalent of
      94      `storage({'a':1, 'b':2, 'c':3})`.
      95
      96      If a `storify` value is a list (e.g. multiple values in a form submission),
      97      `storify` returns the last element of the list, unless the key appears in
      98      `defaults` as a list. Thus:
      99
     100          >>> storify({'a':[1, 2]}).a
     101          2
     102          >>> storify({'a':[1, 2]}, a=[]).a
     103          [1, 2]
     104          >>> storify({'a':1}, a=[]).a
     105          [1]
     106          >>> storify({}, a=[]).a
     107          []
     108
     109      Similarly, if the value has a `value` attribute, `storify will return _its_
     110      value, unless the key appears in `defaults` as a dictionary.
     111
     112          >>> storify({'a':storage(value=1)}).a
     113          1
     114          >>> storify({'a':storage(value=1)}, a={}).a
     115          <Storage {'value': 1}>
     116          >>> storify({}, a={}).a
     117          {}
     118
     119      Optionally, keyword parameter `_unicode` can be passed to convert all values to unicode.
     120
     121          >>> storify({'x': 'a'}, _unicode=True)
     122          <Storage {'x': u'a'}>
     123          >>> storify({'x': storage(value='a')}, x={}, _unicode=True)
     124          <Storage {'x': <Storage {'value': 'a'}>}>
     125          >>> storify({'x': storage(value='a')}, _unicode=True)
     126          <Storage {'x': u'a'}>
     127      """
     128      _unicode = defaults.pop('_unicode', False)
     129      def unicodify(s):
     130          if _unicode and isinstance(s, str): return safeunicode(s)
     131          else: return s
     132
     133      def getvalue(x):
     134          if hasattr(x, 'file') and hasattr(x, 'value'):
     135              return x.value
     136          elif hasattr(x, 'value'):
     137              return unicodify(x.value)
     138          else:
     139              return unicodify(x)
     140
     141      stor = Storage()
     142      for key in requireds + tuple(mapping.keys()):
     143          value = mapping[key]
     144          if isinstance(value, list):
     145              if isinstance(defaults.get(key), list):
     146                  value = [getvalue(x) for x in value]
     147              else:
     148                  value = value[-1]
     149          if not isinstance(defaults.get(key), dict):
     150              value = getvalue(value)
     151          if isinstance(defaults.get(key), list) and not isinstance(value, list):
     152              value = [value]
     153          setattr(stor, key, value)
     154
     155      for (key, value) in defaults.iteritems():
     156          result = value
     157          if hasattr(stor, key):
     158              result = stor[key]
     159          if value == () and not isinstance(result, tuple):
     160              result = (result,)
     161          setattr(stor, key, result)
     162
     163      return stor
     164
     165 -class Counter(storage):
     166      """Keeps count of how many times something is added.
     167
     168          >>> c = counter()
     169          >>> c.add('x')
     170          >>> c.add('x')
     171          >>> c.add('x')
     172          >>> c.add('x')
     173          >>> c.add('x')
     174          >>> c.add('y')
     175          >>> c
     176          <Counter {'y': 1, 'x': 5}>
     177          >>> c.most()
     178          ['x']
     179      """
     180 -    def add(self, n):
     181          self.setdefault(n, 0)
     182          self[n] += 1
     183
     184 -    def most(self):
     185          """Returns the keys with maximum count."""
     186          m = max(self.itervalues())
     187          return [k for k, v in self.iteritems() if v == m]
     188
     189 -    def least(self):
     190          """Returns the keys with mininum count."""
     191          m = min(self.itervalues())
     192          return [k for k, v in self.iteritems() if v == m]
     193
     194 -    def percent(self, key):
     195         """Returns what percentage a certain key is of all entries.
     196
     197             >>> c = counter()
     198             >>> c.add('x')
     199             >>> c.add('x')
     200             >>> c.add('x')
     201             >>> c.add('y')
     202             >>> c.percent('x')
     203             0.75
     204             >>> c.percent('y')
     205             0.25
     206         """
     207         return float(self[key])/sum(self.values())
     208
     209 -    def sorted_keys(self):
     210          """Returns keys sorted by value.
     211
     212               >>> c = counter()
     213               >>> c.add('x')
     214               >>> c.add('x')
     215               >>> c.add('y')
     216               >>> c.sorted_keys()
     217               ['x', 'y']
     218          """
     219          return sorted(self.keys(), key=lambda k: self[k], reverse=True)
     220
     221 -    def sorted_values(self):
     222          """Returns values sorted by value.
     223
     224              >>> c = counter()
     225              >>> c.add('x')
     226              >>> c.add('x')
     227              >>> c.add('y')
     228              >>> c.sorted_values()
     229              [2, 1]
     230          """
     231          return [self[k] for k in self.sorted_keys()]
     232
     233 -    def sorted_items(self):
     234          """Returns items sorted by value.
     235
     236              >>> c = counter()
     237              >>> c.add('x')
     238              >>> c.add('x')
     239              >>> c.add('y')
     240              >>> c.sorted_items()
     241              [('x', 2), ('y', 1)]
     242          """
     243          return [(k, self[k]) for k in self.sorted_keys()]
     244
     245 -    def __repr__(self):
     246          return '<Counter ' + dict.__repr__(self) + '>'
     247
     248  counter = Counter
     249
     250  iters = [list, tuple]
     251  import __builtin__
     252  if hasattr(__builtin__, 'set'):
     253      iters.append(set)
     254  if hasattr(__builtin__, 'frozenset'):
     255      iters.append(set)
     256  if sys.version_info < (2,6): # sets module deprecated in 2.6
     257      try:
     258          from sets import Set
     259          iters.append(Set)
     260      except ImportError:
     261          pass
     262
     263 -class _hack(tuple): pass
     264  iters = _hack(iters)
     265  iters.__doc__ = """
     266  A list of iterable items (like lists, but not strings). Includes whichever
     267  of lists, tuples, sets, and Sets are available in this version of Python.
     268  """
     269
     270 -def _strips(direction, text, remove):
     271      if direction == 'l':
     272          if text.startswith(remove):
     273              return text[len(remove):]
     274      elif direction == 'r':
     275          if text.endswith(remove):
     276              return text[:-len(remove)]
     277      else:
     278          raise ValueError, "Direction needs to be r or l."
     279      return text
     280
     281 -def rstrips(text, remove):
     282      """
     283      removes the string `remove` from the right of `text`
     284
     285          >>> rstrips("foobar", "bar")
     286          'foo'
     287
     288      """
     289      return _strips('r', text, remove)
     290
     291 -def lstrips(text, remove):
     292      """
     293      removes the string `remove` from the left of `text`
     294
     295          >>> lstrips("foobar", "foo")
     296          'bar'
     297
     298      """
     299      return _strips('l', text, remove)
     300
     301 -def strips(text, remove):
     302      """
     303      removes the string `remove` from the both sides of `text`
     304
     305          >>> strips("foobarfoo", "foo")
     306          'bar'
     307
     308      """
     309      return rstrips(lstrips(text, remove), remove)
     310
     311 -def safeunicode(obj, encoding='utf-8'):
     312      r"""
     313      Converts any given object to unicode string.
     314
     315          >>> safeunicode('hello')
     316          u'hello'
     317          >>> safeunicode(2)
     318          u'2'
     319          >>> safeunicode('\xe1\x88\xb4')
     320          u'\u1234'
     321      """
     322      t = type(obj)
     323      if t is unicode:
     324          return obj
     325      elif t is str:
     326          return obj.decode(encoding)
     327      elif t in [int, float, bool]:
     328          return unicode(obj)
     329      else:
     330          if hasattr(obj, '__unicode__'):
     331              return unicode(obj)
     332          else:
     333              return str(obj).decode(encoding)
     334
     335 -def safestr(obj, encoding='utf-8'):
     336      r"""
     337      Converts any given object to utf-8 encoded string.
     338
     339          >>> safestr('hello')
     340          'hello'
     341          >>> safestr(u'\u1234')
     342          '\xe1\x88\xb4'
     343          >>> safestr(2)
     344          '2'
     345      """
     346      if isinstance(obj, unicode):
     347          return obj.encode('utf-8')
     348      elif isinstance(obj, str):
     349          return obj
     350      elif hasattr(obj, 'next') and hasattr(obj, '__iter__'): # iterator
     351          return itertools.imap(safestr, obj)
     352      else:
     353          return str(obj)
     354
     355  # for backward-compatibility
     356  utf8 = safestr
     357
     358 -class TimeoutError(Exception): pass
     359 -def timelimit(timeout):
     360      """
     361      A decorator to limit a function to `timeout` seconds, raising `TimeoutError`
     362      if it takes longer.
     363
     364          >>> import time
     365          >>> def meaningoflife():
     366          ...     time.sleep(.2)
     367          ...     return 42
     368          >>>
     369          >>> timelimit(.1)(meaningoflife)()
     370          Traceback (most recent call last):
     371              ...
     372          TimeoutError: took too long
     373          >>> timelimit(1)(meaningoflife)()
     374          42
     375
     376      _Caveat:_ The function isn't stopped after `timeout` seconds but continues
     377      executing in a separate thread. (There seems to be no way to kill a thread.)
     378
     379      inspired by <http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/473878>
     380      """
     381      def _1(function):
     382          def _2(*args, **kw):
     383              class Dispatch(threading.Thread):
     384                  def __init__(self):
     385                      threading.Thread.__init__(self)
     386                      self.result = None
     387                      self.error = None
     388
     389                      self.setDaemon(True)
     390                      self.start()
     391
     392                  def run(self):
     393                      try:
     394                          self.result = function(*args, **kw)
     395                      except:
     396                          self.error = sys.exc_info()
     397
     398              c = Dispatch()
     399              c.join(timeout)
     400              if c.isAlive():
     401                  raise TimeoutError, 'took too long'
     402              if c.error:
     403                  raise c.error[0], c.error[1]
     404              return c.result
     405          return _2
     406      return _1
     407
     408 -class Memoize:
     409      """
     410      'Memoizes' a function, caching its return values for each input.
     411      If `expires` is specified, values are recalculated after `expires` seconds.
     412      If `background` is specified, values are recalculated in a separate thread.
     413
     414          >>> calls = 0
     415          >>> def howmanytimeshaveibeencalled():
     416          ...     global calls
     417          ...     calls += 1
     418          ...     return calls
     419          >>> fastcalls = memoize(howmanytimeshaveibeencalled)
     420          >>> howmanytimeshaveibeencalled()
     421          1
     422          >>> howmanytimeshaveibeencalled()
     423          2
     424          >>> fastcalls()
     425          3
     426          >>> fastcalls()
     427          3
     428          >>> import time
     429          >>> fastcalls = memoize(howmanytimeshaveibeencalled, .1, background=False)
     430          >>> fastcalls()
     431          4
     432          >>> fastcalls()
     433          4
     434          >>> time.sleep(.2)
     435          >>> fastcalls()
     436          5
     437          >>> def slowfunc():
     438          ...     time.sleep(.1)
     439          ...     return howmanytimeshaveibeencalled()
     440          >>> fastcalls = memoize(slowfunc, .2, background=True)
     441          >>> fastcalls()
     442          6
     443          >>> timelimit(.05)(fastcalls)()
     444          6
     445          >>> time.sleep(.2)
     446          >>> timelimit(.05)(fastcalls)()
     447          6
     448          >>> timelimit(.05)(fastcalls)()
     449          6
     450          >>> time.sleep(.2)
     451          >>> timelimit(.05)(fastcalls)()
     452          7
     453          >>> fastcalls = memoize(slowfunc, None, background=True)
     454          >>> threading.Thread(target=fastcalls).start()
     455          >>> time.sleep(.01)
     456          >>> fastcalls()
     457          9
     458      """
     459 -    def __init__(self, func, expires=None, background=True):
     460          self.func = func
     461          self.cache = {}
     462          self.expires = expires
     463          self.background = background
     464          self.running = {}
     465
     466 -    def __call__(self, *args, **keywords):
     467          key = (args, tuple(keywords.items()))
     468          if not self.running.get(key):
     469              self.running[key] = threading.Lock()
     470          def update(block=False):
     471              if self.running[key].acquire(block):
     472                  try:
     473                      self.cache[key] = (self.func(*args, **keywords), time.time())
     474                  finally:
     475                      self.running[key].release()
     476
     477          if key not in self.cache:
     478              update(block=True)
     479          elif self.expires and (time.time() - self.cache[key][1]) > self.expires:
     480              if self.background:
     481                  threading.Thread(target=update).start()
     482              else:
     483                  update()
     484          return self.cache[key][0]
     485
     486  memoize = Memoize
     487
     488  re_compile = memoize(re.compile) #@@ threadsafe?
     489  re_compile.__doc__ = """
     490  A memoized version of re.compile.
     491  """
     492
     493 -class _re_subm_proxy:
     494 -    def __init__(self):
     495          self.match = None
     496 -    def __call__(self, match):
     497          self.match = match
     498          return ''
     499
     500 -def re_subm(pat, repl, string):
     501      """
     502      Like re.sub, but returns the replacement _and_ the match object.
     503
     504          >>> t, m = re_subm('g(oo+)fball', r'f\\1lish', 'goooooofball')
     505          >>> t
     506          'foooooolish'
     507          >>> m.groups()
     508          ('oooooo',)
     509      """
     510      compiled_pat = re_compile(pat)
     511      proxy = _re_subm_proxy()
     512      compiled_pat.sub(proxy.__call__, string)
     513      return compiled_pat.sub(repl, string), proxy.match
     514
     515 -def group(seq, size):
     516      """
     517      Returns an iterator over a series of lists of length size from iterable.
     518
     519          >>> list(group([1,2,3,4], 2))
     520          [[1, 2], [3, 4]]
     521          >>> list(group([1,2,3,4,5], 2))
     522          [[1, 2], [3, 4], [5]]
     523      """
     524      def take(seq, n):
     525          for i in xrange(n):
     526              yield seq.next()
     527
     528      if not hasattr(seq, 'next'):
     529          seq = iter(seq)
     530      while True:
     531          x = list(take(seq, size))
     532          if x:
     533              yield x
     534          else:
     535              break
     536
     537 -def uniq(seq):
     538     """
     539     Removes duplicate elements from a list.
     540
     541         >>> uniq([1,2,3,1,4,5,6])
     542         [1, 2, 3, 4, 5, 6]
     543     """
     544     seen = set()
     545     result = []
     546     for item in seq:
     547         if item in seen: continue
     548         seen.add(item)
     549         result.append(item)
     550     return result
     551
     552 -def iterview(x):
     553     """
     554     Takes an iterable `x` and returns an iterator over it
     555     which prints its progress to stderr as it iterates through.
     556     """
     557     WIDTH = 70
     558
     559     def plainformat(n, lenx):
     560         return '%5.1f%% (%*d/%d)' % ((float(n)/lenx)*100, len(str(lenx)), n, lenx)
     561
     562     def bars(size, n, lenx):
     563         val = int((float(n)*size)/lenx + 0.5)
     564         if size - val:
     565             spacing = ">" + (" "*(size-val))[1:]
     566         else:
     567             spacing = ""
     568         return "[%s%s]" % ("="*val, spacing)
     569
     570     def eta(elapsed, n, lenx):
     571         if n == 0:
     572             return '--:--:--'
     573         if n == lenx:
     574             secs = int(elapsed)
     575         else:
     576             secs = int((elapsed/n) * (lenx-n))
     577         mins, secs = divmod(secs, 60)
     578         hrs, mins = divmod(mins, 60)
     579
     580         return '%02d:%02d:%02d' % (hrs, mins, secs)
     581
     582     def format(starttime, n, lenx):
     583         out = plainformat(n, lenx) + ' '
     584         if n == lenx:
     585             end = '     '
     586         else:
     587             end = ' ETA '
     588         end += eta(time.time() - starttime, n, lenx)
     589         out += bars(WIDTH - len(out) - len(end), n, lenx)
     590         out += end
     591         return out
     592
     593     starttime = time.time()
     594     lenx = len(x)
     595     for n, y in enumerate(x):
     596         sys.stderr.write('\r' + format(starttime, n, lenx))
     597         yield y
     598     sys.stderr.write('\r' + format(starttime, n+1, lenx) + '\n')
     599
     600 -class IterBetter:
     601      """
     602      Returns an object that can be used as an iterator
     603      but can also be used via __getitem__ (although it
     604      cannot go backwards -- that is, you cannot request
     605      `iterbetter[0]` after requesting `iterbetter[1]`).
     606
     607          >>> import itertools
     608          >>> c = iterbetter(itertools.count())
     609          >>> c[1]
     610          1
     611          >>> c[5]
     612          5
     613          >>> c[3]
     614          Traceback (most recent call last):
     615              ...
     616          IndexError: already passed 3
     617
     618      For boolean test, IterBetter peeps at first value in the itertor without effecting the iteration.
     619
     620          >>> c = iterbetter(iter(range(5)))
     621          >>> bool(c)
     622          True
     623          >>> list(c)
     624          [0, 1, 2, 3, 4]
     625          >>> c = iterbetter(iter([]))
     626          >>> bool(c)
     627          False
     628          >>> list(c)
     629          []
     630      """
     631 -    def __init__(self, iterator):
     632          self.i, self.c = iterator, 0
     633
     634 -    def __iter__(self):
     635          if hasattr(self, "_head"):
     636              yield self._head
     637
     638          while 1:
     639              yield self.i.next()
     640              self.c += 1
     641
     642 -    def __getitem__(self, i):
     643          #todo: slices
     644          if i < self.c:
     645              raise IndexError, "already passed "+str(i)
     646          try:
     647              while i > self.c:
     648                  self.i.next()
     649                  self.c += 1
     650              # now self.c == i
     651              self.c += 1
     652              return self.i.next()
     653          except StopIteration:
     654              raise IndexError, str(i)
     655
     656 -    def __nonzero__(self):
     657          if hasattr(self, "__len__"):
     658              return len(self) != 0
     659          elif hasattr(self, "_head"):
     660              return True
     661          else:
     662              try:
     663                  self._head = self.i.next()
     664              except StopIteration:
     665                  return False
     666              else:
     667                  return True
     668
     669  iterbetter = IterBetter
     670
     671 -def safeiter(it, cleanup=None, ignore_errors=True):
     672      """Makes an iterator safe by ignoring the exceptions occured during the iteration.
     673      """
     674      def next():
     675          while True:
     676              try:
     677                  return it.next()
     678              except StopIteration:
     679                  raise
     680              except:
     681                  traceback.print_exc()
     682
     683      it = iter(it)
     684      while True:
     685          yield next()
     686
     687 -def safewrite(filename, content):
     688      """Writes the content to a temp file and then moves the temp file to
     689      given filename to avoid overwriting the existing file in case of errors.
     690      """
     691      f = file(filename + '.tmp', 'w')
     692      f.write(content)
     693      f.close()
     694      os.rename(f.name, path)
     695
     696 -def dictreverse(mapping):
     697      """
     698      Returns a new dictionary with keys and values swapped.
     699
     700          >>> dictreverse({1: 2, 3: 4})
     701          {2: 1, 4: 3}
     702      """
     703      return dict([(value, key) for (key, value) in mapping.iteritems()])
     704
     705 -def dictfind(dictionary, element):
     706      """
     707      Returns a key whose value in `dictionary` is `element`
     708      or, if none exists, None.
     709
     710          >>> d = {1:2, 3:4}
     711          >>> dictfind(d, 4)
     712          3
     713          >>> dictfind(d, 5)
     714      """
     715      for (key, value) in dictionary.iteritems():
     716          if element is value:
     717              return key
     718
     719 -def dictfindall(dictionary, element):
     720      """
     721      Returns the keys whose values in `dictionary` are `element`
     722      or, if none exists, [].
     723
     724          >>> d = {1:4, 3:4}
     725          >>> dictfindall(d, 4)
     726          [1, 3]
     727          >>> dictfindall(d, 5)
     728          []
     729      """
     730      res = []
     731      for (key, value) in dictionary.iteritems():
     732          if element is value:
     733              res.append(key)
     734      return res
     735
     736 -def dictincr(dictionary, element):
     737      """
     738      Increments `element` in `dictionary`,
     739      setting it to one if it doesn't exist.
     740
     741          >>> d = {1:2, 3:4}
     742          >>> dictincr(d, 1)
     743          3
     744          >>> d[1]
     745          3
     746          >>> dictincr(d, 5)
     747          1
     748          >>> d[5]
     749          1
     750      """
     751      dictionary.setdefault(element, 0)
     752      dictionary[element] += 1
     753      return dictionary[element]
     754
     755 -def dictadd(*dicts):
     756      """
     757      Returns a dictionary consisting of the keys in the argument dictionaries.
     758      If they share a key, the value from the last argument is used.
     759
     760          >>> dictadd({1: 0, 2: 0}, {2: 1, 3: 1})
     761          {1: 0, 2: 1, 3: 1}
     762      """
     763      result = {}
     764      for dct in dicts:
     765          result.update(dct)
     766      return result
     767
     768 -def requeue(queue, index=-1):
     769      """Returns the element at index after moving it to the beginning of the queue.
     770
     771          >>> x = [1, 2, 3, 4]
     772          >>> requeue(x)
     773          4
     774          >>> x
     775          [4, 1, 2, 3]
     776      """
     777      x = queue.pop(index)
     778      queue.insert(0, x)
     779      return x
     780
     781 -def restack(stack, index=0):
     782      """Returns the element at index after moving it to the top of stack.
     783
     784             >>> x = [1, 2, 3, 4]
     785             >>> restack(x)
     786             1
     787             >>> x
     788             [2, 3, 4, 1]
     789      """
     790      x = stack.pop(index)
     791      stack.append(x)
     792      return x
     793
     794 -def listget(lst, ind, default=None):
     795      """
     796      Returns `lst[ind]` if it exists, `default` otherwise.
     797
     798          >>> listget(['a'], 0)
     799          'a'
     800          >>> listget(['a'], 1)
     801          >>> listget(['a'], 1, 'b')
     802          'b'
     803      """
     804      if len(lst)-1 < ind:
     805          return default
     806      return lst[ind]
     807
     808 -def intget(integer, default=None):
     809      """
     810      Returns `integer` as an int or `default` if it can't.
     811
     812          >>> intget('3')
     813          3
     814          >>> intget('3a')
     815          >>> intget('3a', 0)
     816          0
     817      """
     818      try:
     819          return int(integer)
     820      except (TypeError, ValueError):
     821          return default
     822
     823 -def datestr(then, now=None):
     824      """
     825      Converts a (UTC) datetime object to a nice string representation.
     826
     827          >>> from datetime import datetime, timedelta
     828          >>> d = datetime(1970, 5, 1)
     829          >>> datestr(d, now=d)
     830          '0 microseconds ago'
     831          >>> for t, v in {
     832          ...   timedelta(microseconds=1): '1 microsecond ago',
     833          ...   timedelta(microseconds=2): '2 microseconds ago',
     834          ...   -timedelta(microseconds=1): '1 microsecond from now',
     835          ...   -timedelta(microseconds=2): '2 microseconds from now',
     836          ...   timedelta(microseconds=2000): '2 milliseconds ago',
     837          ...   timedelta(seconds=2): '2 seconds ago',
     838          ...   timedelta(seconds=2*60): '2 minutes ago',
     839          ...   timedelta(seconds=2*60*60): '2 hours ago',
     840          ...   timedelta(days=2): '2 days ago',
     841          ... }.iteritems():
     842          ...     assert datestr(d, now=d+t) == v
     843          >>> datestr(datetime(1970, 1, 1), now=d)
     844          'January  1'
     845          >>> datestr(datetime(1969, 1, 1), now=d)
     846          'January  1, 1969'
     847          >>> datestr(datetime(1970, 6, 1), now=d)
     848          'June  1, 1970'
     849          >>> datestr(None)
     850          ''
     851      """
     852      def agohence(n, what, divisor=None):
     853          if divisor: n = n // divisor
     854
     855          out = str(abs(n)) + ' ' + what       # '2 day'
     856          if abs(n) != 1: out += 's'           # '2 days'
     857          out += ' '                           # '2 days '
     858          if n < 0:
     859              out += 'from now'
     860          else:
     861              out += 'ago'
     862          return out                           # '2 days ago'
     863
     864      oneday = 24 * 60 * 60
     865
     866      if not then: return ""
     867      if not now: now = datetime.datetime.utcnow()
     868      if type(now).__name__ == "DateTime":
     869          now = datetime.datetime.fromtimestamp(now)
     870      if type(then).__name__ == "DateTime":
     871          then = datetime.datetime.fromtimestamp(then)
     872      elif type(then).__name__ == "date":
     873          then = datetime.datetime(then.year, then.month, then.day)
     874
     875      delta = now - then
     876      deltaseconds = int(delta.days * oneday + delta.seconds + delta.microseconds * 1e-06)
     877      deltadays = abs(deltaseconds) // oneday
     878      if deltaseconds < 0: deltadays *= -1 # fix for oddity of floor
     879
     880      if deltadays:
     881          if abs(deltadays) < 4:
     882              return agohence(deltadays, 'day')
     883
     884          out = then.strftime('%B %e') # e.g. 'June 13'
     885          if then.year != now.year or deltadays < 0:
     886              out += ', %s' % then.year
     887          return out
     888
     889      if int(deltaseconds):
     890          if abs(deltaseconds) > (60 * 60):
     891              return agohence(deltaseconds, 'hour', 60 * 60)
     892          elif abs(deltaseconds) > 60:
     893              return agohence(deltaseconds, 'minute', 60)
     894          else:
     895              return agohence(deltaseconds, 'second')
     896
     897      deltamicroseconds = delta.microseconds
     898      if delta.days: deltamicroseconds = int(delta.microseconds - 1e6) # datetime oddity
     899      if abs(deltamicroseconds) > 1000:
     900          return agohence(deltamicroseconds, 'millisecond', 1000)
     901
     902      return agohence(deltamicroseconds, 'microsecond')
     903
     904 -def numify(string):
     905      """
     906      Removes all non-digit characters from `string`.
     907
     908          >>> numify('800-555-1212')
     909          '8005551212'
     910          >>> numify('800.555.1212')
     911          '8005551212'
     912
     913      """
     914      return ''.join([c for c in str(string) if c.isdigit()])
     915
     916 -def denumify(string, pattern):
     917      """
     918      Formats `string` according to `pattern`, where the letter X gets replaced
     919      by characters from `string`.
     920
     921          >>> denumify("8005551212", "(XXX) XXX-XXXX")
     922          '(800) 555-1212'
     923
     924      """
     925      out = []
     926      for c in pattern:
     927          if c == "X":
     928              out.append(string[0])
     929              string = string[1:]
     930          else:
     931              out.append(c)
     932      return ''.join(out)
     933
     934 -def commify(n):
     935      """
     936      Add commas to an integer `n`.
     937
     938          >>> commify(1)
     939          '1'
     940          >>> commify(123)
     941          '123'
     942          >>> commify(1234)
     943          '1,234'
     944          >>> commify(1234567890)
     945          '1,234,567,890'
     946          >>> commify(123.0)
     947          '123.0'
     948          >>> commify(1234.5)
     949          '1,234.5'
     950          >>> commify(1234.56789)
     951          '1,234.56789'
     952          >>> commify('%.2f' % 1234.5)
     953          '1,234.50'
     954          >>> commify(None)
     955          >>>
     956
     957      """
     958      if n is None: return None
     959      n = str(n)
     960      if '.' in n:
     961          dollars, cents = n.split('.')
     962      else:
     963          dollars, cents = n, None
     964
     965      r = []
     966      for i, c in enumerate(str(dollars)[::-1]):
     967          if i and (not (i % 3)):
     968              r.insert(0, ',')
     969          r.insert(0, c)
     970      out = ''.join(r)
     971      if cents:
     972          out += '.' + cents
     973      return out
     974
     975 -def dateify(datestring):
     976      """
     977      Formats a numified `datestring` properly.
     978      """
     979      return denumify(datestring, "XXXX-XX-XX XX:XX:XX")
     980
     981
     982 -def nthstr(n):
     983      """
     984      Formats an ordinal.
     985      Doesn't handle negative numbers.
     986
     987          >>> nthstr(1)
     988          '1st'
     989          >>> nthstr(0)
     990          '0th'
     991          >>> [nthstr(x) for x in [2, 3, 4, 5, 10, 11, 12, 13, 14, 15]]
     992          ['2nd', '3rd', '4th', '5th', '10th', '11th', '12th', '13th', '14th', '15th']
     993          >>> [nthstr(x) for x in [91, 92, 93, 94, 99, 100, 101, 102]]
     994          ['91st', '92nd', '93rd', '94th', '99th', '100th', '101st', '102nd']
     995          >>> [nthstr(x) for x in [111, 112, 113, 114, 115]]
     996          ['111th', '112th', '113th', '114th', '115th']
     997
     998      """
     999
    1000      assert n >= 0
    1001      if n % 100 in [11, 12, 13]: return '%sth' % n
    1002      return {1: '%sst', 2: '%snd', 3: '%srd'}.get(n % 10, '%sth') % n
    1003
    1004 -def cond(predicate, consequence, alternative=None):
    1005      """
    1006      Function replacement for if-else to use in expressions.
    1007
    1008          >>> x = 2
    1009          >>> cond(x % 2 == 0, "even", "odd")
    1010          'even'
    1011          >>> cond(x % 2 == 0, "even", "odd") + '_row'
    1012          'even_row'
    1013      """
    1014      if predicate:
    1015          return consequence
    1016      else:
    1017          return alternative
    1018
    1019 -class CaptureStdout:
    1020      """
    1021      Captures everything `func` prints to stdout and returns it instead.
    1022
    1023          >>> def idiot():
    1024          ...     print "foo"
    1025          >>> capturestdout(idiot)()
    1026          'foo\\n'
    1027
    1028      **WARNING:** Not threadsafe!
    1029      """
    1030 -    def __init__(self, func):
    1031          self.func = func
    1032 -    def __call__(self, *args, **keywords):
    1033          from cStringIO import StringIO
    1034          # Not threadsafe!
    1035          out = StringIO()
    1036          oldstdout = sys.stdout
    1037          sys.stdout = out
    1038          try:
    1039              self.func(*args, **keywords)
    1040          finally:
    1041              sys.stdout = oldstdout
    1042          return out.getvalue()
    1043
    1044  capturestdout = CaptureStdout
    1045
    1046 -class Profile:
    1047      """
    1048      Profiles `func` and returns a tuple containing its output
    1049      and a string with human-readable profiling information.
    1050
    1051          >>> import time
    1052          >>> out, inf = profile(time.sleep)(.001)
    1053          >>> out
    1054          >>> inf[:10].strip()
    1055          'took 0.0'
    1056      """
    1057 -    def __init__(self, func):
    1058          self.func = func
    1059 -    def __call__(self, *args): ##, **kw):   kw unused
    1060          import hotshot, hotshot.stats, os, tempfile ##, time already imported
    1061          f, filename = tempfile.mkstemp()
    1062          os.close(f)
    1063
    1064          prof = hotshot.Profile(filename)
    1065
    1066          stime = time.time()
    1067          result = prof.runcall(self.func, *args)
    1068          stime = time.time() - stime
    1069          prof.close()
    1070
    1071          import cStringIO
    1072          out = cStringIO.StringIO()
    1073          stats = hotshot.stats.load(filename)
    1074          stats.stream = out
    1075          stats.strip_dirs()
    1076          stats.sort_stats('time', 'calls')
    1077          stats.print_stats(40)
    1078          stats.print_callers()
    1079
    1080          x =  '\n\ntook '+ str(stime) + ' seconds\n'
    1081          x += out.getvalue()
    1082
    1083          # remove the tempfile
    1084          try:
    1085              os.remove(filename)
    1086          except IOError:
    1087              pass
    1088
    1089          return result, x
    1090
    1091  profile = Profile
    1092
    1093
    1094  import traceback
    1095  # hack for compatibility with Python 2.3:
    1096  if not hasattr(traceback, 'format_exc'):
    1097      from cStringIO import StringIO
    1098 -    def format_exc(limit=None):
    1099          strbuf = StringIO()
    1100          traceback.print_exc(limit, strbuf)
    1101          return strbuf.getvalue()
    1102      traceback.format_exc = format_exc
    1103
    1104 -def tryall(context, prefix=None):
    1105      """
    1106      Tries a series of functions and prints their results.
    1107      `context` is a dictionary mapping names to values;
    1108      the value will only be tried if it's callable.
    1109
    1110          >>> tryall(dict(j=lambda: True))
    1111          j: True
    1112          ----------------------------------------
    1113          results:
    1114             True: 1
    1115
    1116      For example, you might have a file `test/stuff.py`
    1117      with a series of functions testing various things in it.
    1118      At the bottom, have a line:
    1119
    1120          if __name__ == "__main__": tryall(globals())
    1121
    1122      Then you can run `python test/stuff.py` and get the results of
    1123      all the tests.
    1124      """
    1125      context = context.copy() # vars() would update
    1126      results = {}
    1127      for (key, value) in context.iteritems():
    1128          if not hasattr(value, '__call__'):
    1129              continue
    1130          if prefix and not key.startswith(prefix):
    1131              continue
    1132          print key + ':',
    1133          try:
    1134              r = value()
    1135              dictincr(results, r)
    1136              print r
    1137          except:
    1138              print 'ERROR'
    1139              dictincr(results, 'ERROR')
    1140              print '   ' + '\n   '.join(traceback.format_exc().split('\n'))
    1141
    1142      print '-'*40
    1143      print 'results:'
    1144      for (key, value) in results.iteritems():
    1145          print ' '*2, str(key)+':', value
    1146
    1147 -class ThreadedDict:
    1148      """
    1149      Thread local storage.
    1150
    1151          >>> d = ThreadedDict()
    1152          >>> d.x = 1
    1153          >>> d.x
    1154          1
    1155          >>> import threading
    1156          >>> def f(): d.x = 2
    1157          ...
    1158          >>> t = threading.Thread(target=f)
    1159          >>> t.start()
    1160          >>> t.join()
    1161          >>> d.x
    1162          1
    1163      """
    1164 -    def __getattr__(self, key):
    1165          return getattr(self._getd(), key)
    1166
    1167 -    def __setattr__(self, key, value):
    1168          return setattr(self._getd(), key, value)
    1169
    1170 -    def __delattr__(self, key):
    1171          return delattr(self._getd(), key)
    1172
    1173 -    def __hash__(self):
    1174          return id(self)
    1175
    1176 -    def _getd(self):
    1177          t = threading.currentThread()
    1178          if not hasattr(t, '_d'):
    1179              # using __dict__ of thread as thread local storage
    1180              t._d = {}
    1181
    1182          # there could be multiple instances of ThreadedDict.
    1183          # use self as key
    1184          if self not in t._d:
    1185              t._d[self] = storage()
    1186          return t._d[self]
    1187
    1188  threadeddict = ThreadedDict
    1189
    1190 -def autoassign(self, locals):
    1191      """
    1192      Automatically assigns local variables to `self`.
    1193
    1194          >>> self = storage()
    1195          >>> autoassign(self, dict(a=1, b=2))
    1196          >>> self
    1197          <Storage {'a': 1, 'b': 2}>
    1198
    1199      Generally used in `__init__` methods, as in:
    1200
    1201          def __init__(self, foo, bar, baz=1): autoassign(self, locals())
    1202      """
    1203      for (key, value) in locals.iteritems():
    1204          if key == 'self':
    1205              continue
    1206          setattr(self, key, value)
    1207
    1208 -def to36(q):
    1209      """
    1210      Converts an integer to base 36 (a useful scheme for human-sayable IDs).
    1211
    1212          >>> to36(35)
    1213          'z'
    1214          >>> to36(119292)
    1215          '2k1o'
    1216          >>> int(to36(939387374), 36)
    1217          939387374
    1218          >>> to36(0)
    1219          '0'
    1220          >>> to36(-393)
    1221          Traceback (most recent call last):
    1222              ...
    1223          ValueError: must supply a positive integer
    1224
    1225      """
    1226      if q < 0: raise ValueError, "must supply a positive integer"
    1227      letters = "0123456789abcdefghijklmnopqrstuvwxyz"
    1228      converted = []
    1229      while q != 0:
    1230          q, r = divmod(q, 36)
    1231          converted.insert(0, letters[r])
    1232      return "".join(converted) or '0'
    1233
    1234
    1235  r_url = re_compile('(?<!\()(http://(\S+))')
    1236 -def safemarkdown(text):
    1237      """
    1238      Converts text to HTML following the rules of Markdown, but blocking any
    1239      outside HTML input, so that only the things supported by Markdown
    1240      can be used. Also converts raw URLs to links.
    1241
    1242      (requires [markdown.py](http://webpy.org/markdown.py))
    1243      """
    1244      from markdown import markdown
    1245      if text:
    1246          text = text.replace('<', '&lt;')
    1247          # TODO: automatically get page title?
    1248          text = r_url.sub(r'<\1>', text)
    1249          text = markdown(text)
    1250          return text
    1251
    1252 -def sendmail(from_address, to_address, subject, message, headers=None, **kw):
    1253      """
    1254      Sends the email message `message` with mail and envelope headers
    1255      for from `from_address_` to `to_address` with `subject`.
    1256      Additional email headers can be specified with the dictionary
    1257      `headers.
    1258
    1259      Optionally cc, bcc and attachments can be specified as keyword arguments.
    1260      Attachments must be an iterable and each attachment can be either a
    1261      filename or a file object or a dictionary with filename, content and
    1262      optionally content_type keys.
    1263
    1264      If `web.config.smtp_server` is set, it will send the message
    1265      to that SMTP server. Otherwise it will look for
    1266      `/usr/sbin/sendmail`, the typical location for the sendmail-style
    1267      binary. To use sendmail from a different path, set `web.config.sendmail_path`.
    1268      """
    1269      attachments = kw.pop("attachments", [])
    1270      mail = _EmailMessage(from_address, to_address, subject, message, headers, **kw)
    1271
    1272      for a in attachments:
    1273          if isinstance(a, dict):
    1274              mail.attach(a['filename'], a['content'], a.get('content_type'))
    1275          elif hasattr(a, 'read'): # file
    1276              filename = os.path.basename(getattr(a, "name", ""))
    1277              content_type = getattr(a, 'content_type', None)
    1278              mail.attach(filename, a.read(), content_type)
    1279          elif isinstance(a, basestring):
    1280              f = open(a, 'rb')
    1281              content = f.read()
    1282              f.close()
    1283              filename = os.path.basename(a)
    1284              mail.attach(filename, content, None)
    1285          else:
    1286              raise ValueError, "Invalid attachment: %s" % repr(a)
    1287
    1288      mail.send()
    1289
    1290 -class _EmailMessage:
    1291 -    def __init__(self, from_address, to_address, subject, message, headers=None, **kw):
    1292          def listify(x):
    1293              if not isinstance(x, list):
    1294                  return [safestr(x)]
    1295              else:
    1296                  return [safestr(a) for a in x]
    1297
    1298          subject = safestr(subject)
    1299          message = safestr(message)
    1300
    1301          from_address = safestr(from_address)
    1302          to_address = listify(to_address)
    1303          cc = listify(kw.get('cc', []))
    1304          bcc = listify(kw.get('bcc', []))
    1305          recipients = to_address + cc + bcc
    1306
    1307          import email.Utils
    1308          self.from_address = email.Utils.parseaddr(from_address)[1]
    1309          self.recipients = [email.Utils.parseaddr(r)[1] for r in recipients]
    1310
    1311          self.headers = dictadd({
    1312            'From': from_address,
    1313            'To': ", ".join(to_address),
    1314            'Subject': subject
    1315          }, headers or {})
    1316
    1317          if cc:
    1318              self.headers['Cc'] = ", ".join(cc)
    1319
    1320          self.message = self.new_message()
    1321          self.message.add_header("Content-Transfer-Encoding", "7bit")
    1322          self.message.add_header("Content-Disposition", "inline")
    1323          self.message.add_header("MIME-Version", "1.0")
    1324          self.message.set_payload(message, 'utf-8')
    1325          self.multipart = False
    1326
    1327 -    def new_message(self):
    1328          from email.Message import Message
    1329          return Message()
    1330
    1331 -    def attach(self, filename, content, content_type=None):
    1332          if not self.multipart:
    1333              msg = self.new_message()
    1334              msg.add_header("Content-Type", "multipart/mixed")
    1335              msg.attach(self.message)
    1336              self.message = msg
    1337              self.multipart = True
    1338
    1339          import mimetypes
    1340          try:
    1341              from email import encoders
    1342          except:
    1343              from email import Encoders as encoders
    1344
    1345          content_type = content_type or mimetypes.guess_type(filename)[0] or "applcation/octet-stream"
    1346
    1347          msg = self.new_message()
    1348          msg.set_payload(content)
    1349          msg.add_header('Content-Type', content_type)
    1350          msg.add_header('Content-Disposition', 'attachment', filename=filename)
    1351
    1352          if not content_type.startswith("text/"):
    1353              encoders.encode_base64(msg)
    1354
    1355          self.message.attach(msg)
    1356
    1357 -    def prepare_message(self):
    1358          for k, v in self.headers.iteritems():
    1359              if k.lower() == "content-type":
    1360                  self.message.set_type(v)
    1361              else:
    1362                  self.message.add_header(k, v)
    1363
    1364          self.headers = {}
    1365
    1366 -    def send(self):
    1367          try:
    1368              import webapi
    1369          except ImportError:
    1370              webapi = Storage(config=Storage())
    1371
    1372          self.prepare_message()
    1373          message_text = self.message.as_string()
    1374
    1375          if webapi.config.get('smtp_server'):
    1376              server = webapi.config.get('smtp_server')
    1377              port = webapi.config.get('smtp_port', 0)
    1378              username = webapi.config.get('smtp_username')
    1379              password = webapi.config.get('smtp_password')
    1380              debug_level = webapi.config.get('smtp_debuglevel', None)
    1381              starttls = webapi.config.get('smtp_starttls', False)
    1382
    1383              import smtplib
    1384              smtpserver = smtplib.SMTP(server, port)
    1385
    1386              if debug_level:
    1387                  smtpserver.set_debuglevel(debug_level)
    1388
    1389              if starttls:
    1390                  smtpserver.ehlo()
    1391                  smtpserver.starttls()
    1392                  smtpserver.ehlo()
    1393
    1394              if username and password:
    1395                  smtpserver.login(username, password)
    1396
    1397              smtpserver.sendmail(self.from_address, self.recipients, message_text)
    1398              smtpserver.quit()
    1399          else:
    1400              sendmail = webapi.config.get('sendmail_path', '/usr/sbin/sendmail')
    1401
    1402              assert not self.from_address.startswith('-'), 'security'
    1403              for r in self.recipients:
    1404                  assert not r.startswith('-'), 'security'
    1405
    1406              cmd = [sendmail, '-f', self.from_address] + self.recipients
    1407
    1408              if subprocess:
    1409                  p = subprocess.Popen(cmd, stdin=subprocess.PIPE)
    1410                  p.stdin.write(message_text)
    1411                  p.stdin.close()
    1412                  p.wait()
    1413              else:
    1414                  i, o = os.popen2(cmd)
    1415                  i.write(message)
    1416                  i.close()
    1417                  o.close()
    1418                  del i, o
    1419
    1420 -    def __repr__(self):
    1421          return "<EmailMessage>"
    1422
    1423 -    def __str__(self):
    1424          return self.message.as_string()
    1425
    1426  if __name__ == "__main__":
    1427      import doctest
    1428      doctest.testmod()
    1429