ZergCreepCode

 """Content Routing Utilities Module
 
 This module collects code that helps manipulate data, and route it to
 other environments.
 
 Some functionality requires other modules, special configuration.
 * To upload text to OddMuse wiki, you need the OddMuse wikiput script.
   Download it from:
   http://www.oddmuse.org/cgi-bin/wiki/Automatic_Posting_and_Uploading
 * To upload files to OddMuse wiki, you need the OddMuse wikiupload
   script. Download it from:
   http://www.oddmuse.org/cgi-bin/wiki/Automatic_Posting_and_Uploading
 * To use anything involving RSS, you need feedparser.
 * If you use Inkscape's ability to make PNG files from SVG, you'll need
   to configure the "where_is_inkscape" variable.
 
 .where_is_inkscape  -- location of inkscape.exe (CONFIGURE THIS)
 .memory_store  -- where to store memory (CONFIGURE THIS)
 .ok_folder_characters  -- characters that can appear in folder names
 NullFile  -- the null file class
 .null_file  -- the null file object
 open2  -- replacement open function
 remove_extension  -- take off .whatever from filename
 remove_path  -- take off leading path from filename
 remove_path_and_extension  -- take off leading path, .extension
 .camelcase_re  -- regex for CamelCase
 .halfcamelcase_re  -- regex for Halfcamelcase
 camelcase_p  -- is a string CamelCase?
 halfcamelcase_p  -- is a string Halfcamelcase?
 make_sure_its_camelcase  -- (as it says)
 domain_name_from_url  -- extract a domain name from a URL
 distort_domain_name  -- make 8 characters from a domain name
 brand_dictionary  -- mark a dictionary with source information
 TextFile  -- (as it says)
 PngImageFile  -- (as it says)
 InkscapeSvgImageFile  -- (as it says)
 Folder  -- (as it says)
 memory_store_folder  -- get a Folder from the memory store
 folder_associated_with_url  -- get a Folder associated with a URL
 ImageFile  -- (image file, type detected by filename extension)
 ObjectFile  -- (object file, type detected by filename extension)
 inkscape_create_png_from_svg  -- create a .png from a .svg
 send_both_png_and_svg  -- given a .svg, also send a .png
 send_everything_in_folder  -- send all files found in a folder
 OddMuseWiki  -- an entire OddMuse wiki
 OddMuseWikiPage  -- a single page on the OddMuse wiki
 RssFeed  -- an RSS feed, remembering what was last seen
 .community_wiki  -- LionKimbro's favorite wiki (CHECK THIS)
 """
 
 import os
 import os.path
 import re
 import time
 
 import shutil
 import tempfile
 import subprocess
 import feedparser
 import cPickle as pickle
 import urllib2
 import urlparse
 
 import wikiput
 import wikiupload
 
 
 where_is_inkscape = "C:\\Program Files\\Inkscape\\inkscape.exe"
 
 memory_store = "C:\\memory"  # Must be a folder, specified from root
 
 
 ok_folder_characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + \
                        "abcdefghijklmnopqrstuvwxyz" + \
                        "0123456789"
 
 
 class NullFile:
 
     """A faux file that pretends to be an empty file.
 
     Expand this class as necessary.
 
     Access it through the primary instane, "null_file".
     """
 
     def read(self):
         return ""
 
 
 null_file = NullFile()
 
 
 def open2(filename, mode="r", raise_exception=True,
           default_to_none=False, default=None):
     """Open a file. (Lion's version.)
 
     Specify what to do if the file is not found, as part of the call.
     If you specify a default other than None, it will be returned.
     If you set default_to_none=True, then it will return None.
     
     If there is no default, it will raise an exception, unless you set
     raise_exception to False.
 
     Example uses:
       open2("foo.txt")  # Raises exception
       open2("eraseme.txt", raise_exception=True)  # Raises exception
       open2("eraseme.txt", raise_exception=False)  # Returns None
       open2("eraseme.txt", default_to_none=True)  # Returns None
       open2("bar.txt", default=null_file)  # Returns null_file
 
     Why are there redundant expressions? In order to make the code
     plain, and more expressive, about what the programmer expects, and
     wants other  help the code 
     """
     try:
         return open(filename, mode)
     except IOError, err:
         if default is not None:
             return default
         elif default_to_none == True:
             return None
         elif raise_exception == True:
             raise err
     return None
 
 
 def remove_extension(filename):
     """Remove the .whatever from the end of the filename.
 
     Assume the extension starts with a period, of course.
     """
     index = filename.rfind(".")
     path_sep_index = filename.rfind(os.path.sep)
     if path_sep_index > index:
         return filename  # No extension, period part of folder name
     if index == -1:
         return filename  # No extension, because there's no period
     return filename[:index]
 
 
 def remove_path(filename):
     """Remove the path leading up to the end of the filename.
 
     If the filename ends in the path separator, it returns a blank
     string.
     """
     index = filename.rfind(os.path.sep)
     if index == -1:
         return filename  # No path information
     return filename[index+1:]
 
 
 def remove_path_and_extension(filename):
     """Both remove the path, and the extension, from a filepath-name."""
     return remove_path(remove_extension(filename))
 
 
 camelcase_re = re.compile("[A-Z]+[a-z]+[A-Z]+[a-z]+[A-Za-z]*")
 halfcamelcase_re = re.compile("[A-Z]+[a-z]+[A-Za-z]*")
 
 
 def camelcase_p(name):
     """Return true if the name is camelcase, false otherwise."""
     return camelcase_re.match(name) is not None
 
 
 def halfcamelcase_p(name):
     """Return true if the name could be the beginning of a CamelCase."""
     return halfcamelcase_re.match(name) is not None
 
 
 def make_sure_its_camelcase(name):
     if not camelcase_p(name):
         raise ValueError("%s isn't camelcase, but it should be" % name)
 
 
 def domain_name_from_url(url):
     """Remove http:// and any trailing info from a URL.
     
     For example, it turns "http://foo.bar/" into "foo.bar".
     """
     netloc = urlparse.urlsplit(url)
     if "@" in netloc:
         # remove user:passwd (not often used)
         netloc = netloc.split("@")[-1]
     if ":" in netloc:
         # remove port
         netloc = netloc.split(":")[0]
     return netloc
 
 
 def distort_domain_name(name, count=1, prefix=""):
     """Distort a domain name into an 8 character name.
 
     The 8 character name should be suitable for the filesystem. It
     should be composed entirely of letters and numbers.
 
     To help differentiate similar domain names, you can supply a
     count.  Two names that are the same, but with different counts, will
     look different.
 
     You can also supply a (hopefully short!) prefix, that will be worked
     into the 8 characters. A ValueError is raised if the prefix is 8 or
     more characters.
     """
     if len(prefix) >= 8:
         raise ValueError("Prefix completely consumes naming " \
                          "distortion.\n That means the original name " \
                          "has no contribution to the distorted name.")
     if name.startswith("www."):
         name = name[4:]
     name = "".join([c for c in name if c in ok_folder_characters])
     name = prefix + name[:8-len(prefix)]
     if count == 0:
         raise ValueError("Count begins at 1, not 0.\n" \
                          "This is just so that the naming makes more" \
                          " sense.")
     if count == 1:
         return name
     count_string = str(count)
     count_length = len(count_string)
     return name[:-count_length] + count_string
 
 
 def brand_dictionary(D, brand, ignore_none=False):
     """Brand a dictionary to a particular source.
     
     Return the dictionary.
     """
     if D is None and ignore_none:
         return D  # Effectively, return None
     D["BRAND"] = brand
     return D
 
 
 class TextFile:
     
     """Represent a text file.
     
     DOC
     
     DOC
     """
     
     def __init__(self, filename):
         """Remember the filename of the text file."""
         self.filename = filename
 
 
 class PngImageFile:
 
     """Represent a PNG image file.
 
     DOC
 
     DOC
     """
 
     is_an_image_file = True
 
     def __init__(self, filename):
         """Remember the filename of the PNG image file."""
         self.filename = filename
 
 
 class InkscapeSvgImageFile:
 
     """Represent an Inkscape SVG image file.
 
     DOC
 
     DOC
     """
 
     is_an_image_file = True
 
     def __init__(self, filename):
         """Remember the filename of the Inkscape SVG image file."""
         self.filename = filename
 
 
 class Folder:
     
     """Represent a folder.
 
     DOC
 
     .path  -- folder path, without a trailing slash
     .receive  -- copy things into this folder
     DOC
     """
 
     def __init__(self, path, create=False, throw_exception=True):
         """Remember the path of the folder.
 
         If the folder doesn't exist:
         If create is True, create the folder.
         If throw_exception is True, raise a ValueError.
         Otherwise, just keep going along, without a material folder.
         """
         if len(path) == 0:
             raise ValueError("A folders path can't be zero length.")
         if not os.path.exists(path):
             if create:
                 os.mkdir(path)
             elif throw_exception:
                 raise ValueError(path + " not found")
         if path[-1] == os.path.sep:
             path = path[:-1]
         self.path = path
 
     def ensure_exists(self):
         """If the folder doesn't exist yet, make it exist."""
         if not os.path.exists(self.path):
             os.mkdir(self.path)
     
     def __getitem__(self, filename):
         """DOC"""
         f = open(self.path + os.path.sep + filename, "r")
         content = f.read()
         f.close()
         return content
 
     def get(self, filename, default=None):
         """DOC"""
         f = open2(self.path + os.path.sep + filename, "r",
                   default_to_none=True)
         if f == None:
             return default
         content = f.read()
         f.close()
         return content
     
     def __setitem__(self, filename, value):
         """DOC"""
         f = open(self.path + os.path.sep + filename, "w")
         f.write(str(value))
         f.close()
 
     def __delitem__(self, filename):
         os.remove(self.path + os.path.sep + filename)
 
     def __contains__(self, filename):
         """If a file of that name in the folder?"""
         return os.path.exists(self.path + os.path.sep + filename)
 
     def keys(self):
         """Return a list of the files in the folder."""
         return os.listdir(self.path)
     
     def receive(self, something, filename=None, overwrite=False,
                 **otherstuff):
         """Put something into the folder.
 
         "filename" is a new filename, to use in the folder. If it's
         not specified, one will be intelligently constructed from the
         something.
 
         Set "overwrite" to True, if you are okay with overwriting
         files, and don't want to get an exception.
         """
         if something.__class__.__name__ in ["TextFile",
                                             "PngImageFile",
                                             "InkscapeSvgImageFile",]:
             if filename is None:
                 filename = remove_path_and_extension(something.filename)
             shutil.copy(something.filename,
                         self.path + os.path.sep + filename)
         elif something.__class__.__name__ == "OddMuseWikiPage":
             if not overwrite:
                 if something.pagename in os.listdir(self.path):
                     raise ValueError("Wiki page %s is already in this" \
                                      " folder.\nSet overwrite=True if" \
                                      " you want to overwrite, anyways.")
             # Get the content first, so that if there's a failure, we
             # get the exception, and don't erase a file that may be
             # presently there.
             content = something.raw_content(raise_exception=True)
             # Assuming everything was allright, no exceptions raised, we
             # can now write the page into the folder.
             if filename is None:
                 filename = something.pagename + ".txt"
             f = open(self.path + os.path.sep + filename, "w")
             f.write(content)
             f.close()
         elif something.__class__.__name__ == "Folder":
             if filename is None:
                 filename = remove_path_and_extension(something.filename)
             shutil.copytree(something.filename,
                             self.path + os.path.sep + filename)
         elif something.__class__.__name__ == "str":
             if filename is None:
                 # Check for the first filename not taken, of the form
                 # "notexxxx.txt".  For example, first check for
                 # "note0000.txt".  If it's not there, try for
                 # "note0001.txt".  If that's not there, try for
                 # "note0002.txt".  And so on, and so forth.
                 x = 0
                 while filename is None:
                     filename = "note%04d.txt" % x
                     if filename in os.listdir(self.path):
                         filename = None
             f = open(self.path + os.path.sep + filename, "w")
             f.write(something)
             f.close()
         else:
             raise ValueError("I don't know how to receive %s." \
                              % something)
 
 
 def memory_store_folder(relative_path, create=False,
                            throw_exception=True):
     """Get a folder from inside the memory store."""
     full_path = memory_store + os.path.sep + relative_path
     return Folder(full_path, create=create,
                      throw_exception=throw_exception)
 
 
 def folder_associated_with_url(url, prefix="", create=False,
                                   default=None, default_to_none=False,
                                   throw_exception=True):
     """Return a Folder associated with the given URL.
 
     See "Folder.__init__" for details about arguments.
     The "prefix" argument is different, it is an optional prefix to the
     folder name, and is passed to distort_domain_name.
 
     distort_domain_name is used to generate a folder name that
     somewhat resembles the URL.  Every URL is guaranteed a unique
     folder.
     """
     domain_name = domain_name_from_url(url)
     count = 1
     distorted_name = distort_domain_name(domain_name, count,
                                          prefix=prefix)
     found = False
     while found == False:
         try:
             folder = memory_store_folder(distorted_name,
                                          throw_exception=True)
             # The folder was found.
             if "represents.txt" in folder:
                 if folder["represents.txt"] == url:
                     return folder
             count = count + 1
             distorted_name = distort_domain_name(domain_name, count,
                                                  prefix=prefix)
         except ValueError, err:
             # There was no folder by that name.
             if create:
                 folder = memory_store_folder(distorted_name,
                                              create=True)
                 folder["represents.txt"] = url
                 return folder
             elif default is not None:
                 return default
             elif default_to_none:
                 return None
             elif throw_exception:
                 raise err
             else:
                 return None
 
 
 def ImageFile(filename):
     """Faux class __init__; Return a properly instantiated object."""
     if filename.endswith(".svg"):
         # The assumption that the .svg file is an Inkscape SVG file
         # meets my purposes now, but may be problematic later on.
         #
         # If you're fixing this, there are various things you can do
         # with the XML file to see if it is from Inkscape or not.
         return InkscapeSvgImageFile(filename)
     elif filename.endswith(".png"):
         return PngImageFile(filename)
     raise ValueError("I don't recognize that image file type;\n" \
                      "I go by extension, and I don't recognize the\n" \
                      "extension on: %s\n" % filename)
 
 
 def ObjectFile(filename):
     """Create an object, based on the form of the filename.
     
     Faux class __init__, like ImageFile, but not specific to images.
     
     If the filename ends in:
     .txt  -- TextFile
     .png  -- PngImageFile
     .svg  -- InkscapeSvgImageFile
     /     -- Folder
 
     Raise a ValueError exception when it's none of the above.
     """
     if filename.endswith(".txt"):
         return TextFile(filename)
     if filename.endswith(".png"):
         return PngImageFile(filename)
     if filename.endswith(".svg"):
         return InkscapeSvgImageFile(filename)
     if filename.endswith("/"):
         return Folder(filename)
     raise ValueError("Can't decypher what sort of file %s is." \
                      % filename)
 
 
 def inkscape_create_png_from_svg(inkscape_svg_image, png_name):
     """Create a .PNG file from an .SVG file, using Inkscape.
 
     The .PNG file is created in a temporary folder, the folder
     name of which is returned as the second value in a two-tuple.
 
     The first thing returned is the PngImageFile.
 
     The png_name should be free of ".png", and free of any path
     information. Otherwise, you will likely have some problems.
     """
     tmpdir_path = tempfile.mkdtemp()
     png_filename = tmpdir_path + os.path.sep + png_name + ".png"
     # It took a long time to figure out the exact command string that
     # would work, and the exact Python module & function to use.  If you
     # just use "os.system(command)", for example, this will fail.  So,
     # be very careful, if you adjust this process.
     command = '"%s" --export-png "%s" "%s"' \
               % (where_is_inkscape, png_filename,
                  inkscape_svg_image.filename)
     result = subprocess.Popen(command).wait()
     if result != 0:
         if not os.path.exists(where_is_inkscape):
             raise OSError("You're trying to create a PNG from an SVG," \
                           " but I can't get to:\n" \
                           "  %s\n" \
                           "You have to set .where_is_inkscape in the" \
                           " inkutil module (inkutil.py),\n" \
                           "or give me proper rights to it.\n" % \
                           where_is_inkscape)
         raise OSError(result, "Inkscape command failed", command)
     return (PngImageFile(png_filename), tmpdir_path)
 
 
 def send_both_png_and_svg(inkscape_svg_image, recipient,
                           should_be_camelcase=True,
                           **otherstuff):
     """Send both .PNG & .SVG source to the recipient.
 
     The .PNG image is called "FooBarImage", and the .SVG image is called
     "FooBarSource."
 
     The recipient needs to have a special method,
     ".receive_approved_svg(inkscape_svg_image)", to receive the SVG
     by. The special method is there, because a lot of recipients will
     receive an SVG file, and then call this function, and then we're
     going to send an SVG file right back to them. So, to avoid an
     infinite loop, we send it back in the confidence of this special
     function.
     """
     filename = inkscape_svg_image.filename
     if filename[-4:].lower() != ".svg":
         raise ValueError("Inkscape SVG filename should end with .svg.")
     if "pagename" in otherstuff:
         pagename = otherstuff["pagename"]
     else:
         pagename = remove_path_and_extension(filename)
     if should_be_camelcase:
         if not halfcamelcase_p(pagename):
             raise ValueError("%s isn't the start of a camelcase name," \
                              " and it should be.\n" \
                              "(The original filename was: %s)\n" %
                              (pagename, inkscape_svg_image.filename,))
     args = otherstuff.copy()
     args["pagename"] = pagename + "Source"
     recipient.__class__.receive_approved_svg(recipient,
                                              inkscape_svg_image, **args)
     (png_image,
      tmpdir_path) = inkscape_create_png_from_svg(inkscape_svg_image,
                                                  pagename + "Image")
     args["pagename"] = pagename + "Image"
     recipient.__class__.receive(recipient, png_image, **args)
     os.remove(png_image.filename)  # Clean up
     os.rmdir(tmpdir_path)  # Clean up
 
 
 def send_everything_in_folder(folder, recipient, **otherstuff):
     """Send everything in the folder to the recipient.
 
     This function turns everything into an object with ObjectFile,
     before sending it to the recipient.
     """
     for individual_filename in os.listdir(folder.path):
         something = ObjectFile(folder.path + os.path.sep
                                + individual_filename)
         receive.__class__.receive(something, **otherstuff)
 
 
 class OddMuseWiki:
 
     """Represent an OddMuse wiki.
 
     DOC
 
     DOC
     """
 
     def __init__(self, url_prefix):
         """Initialize the OddMuseWiki with it's URL prefix.
 
         The URL prefix probably ends with a slash, and it's what you put
         in front of a page's name, for normal, regular, reading.
 
         ex: For CommunityWiki, it's: "http://www.communitywiki.org/"
         """
         self.url_prefix = url_prefix
         self.default_username = "UnnamedScript"
         self.default_editsummary = "Programmaticly edited."
     
     def testing_page(self):
         """Return a sendable object representing a test page."""
         return self.page("SandBox")
     
     def page(self, pagename):
         """Return a particular page from the wiki."""
         return OddMuseWikiPage(self, pagename)
     
     def rss_feed(self):
         """Return the RecentChanges RSS feed for the wiki."""
         return RssFeed(self.url_prefix + "?action=rss")
 
     def has_new_entries(self, **args):
         """DOC"""
         return self.rss_feed().has_new_entries(**args)
     
     def get_one_new_entry(self, **args):
         """DOC"""
         entry = self.rss_feed().get_one_new_entry(**args)
         if entry is None:
             return None
         return self.page(entry["title"])
     
     def get_all_new_entries(self, **args):
         """DOC"""
         return [self.page(x["title"]) for x in
                 self.rss_feed().get_one_new_entry(**args)]
     
     def receive_approved_svg(self, inkscape_svg_image, **otherstuff):
         """A special function; Put an SVG image onto the OddMuseWiki.
 
         This is called specially by send_both_png_and_svg, and isn't
         intended to be called by anything else.
 
         The pagename is supposed to be supplied by
         send_both_png_and_svg.
         """
         if "pagename" not in otherstuff:
             raise ValueError("send_both_png_and_svg was supposed to" \
                              " send \"pagename\", but it didn't.")
         editsummary = otherstuff.get("editsummary",
                                      self.default_editsummary)
         username = otherstuff.get("username",
                                   self.default_username)
         wikiupload.wikiput(self.url_prefix + otherstuff["pagename"],
                            inkscape_svg_image.filename,
                            editsummary, username,
                            type="image/svg+xml")
     
     def receive(self, something, username=None, editsummary=None,
                 pagename=None, **otherstuff):
         """Place a something onto the OddMuseWiki page.
 
         If no page name is specified, there's an effort to infer it from
         whatever the "something" is.
         """
         if username is None:
             username = self.default_username
         if editsummary is None:
             if hasattr(something, "is_an_image_file"):
                 editsummary = "Uploading image."
             else:
                 editsummary = self.default_editsummary
         if something.__class__.__name__ == "InkscapeSvgImageFile":
             if pagename is None:
                 pagename = remove_path_and_extension(something.filename)
             send_both_png_and_svg(something, self,
                                   should_be_camelcase=True,
                                   username=username,
                                   editsummary=editsummary,
                                   pagename=pagename)
         elif something.__class__.__name__ == "PngImageFile":
             if pagename is None:
                 pagename = remove_path_and_extension(something.filename)
             make_sure_its_camelcase(pagename)
             wikiupload.wikiput(self.url_prefix + pagename,
                                something.filename,
                                editsummary, username,
                                type="image/png")
         elif something.__class__.__name__ == "TextFile":
             if pagename is None:
                 pagename = remove_path_and_extension(something.filename)
             make_sure_its_camelcase(pagename)
             wikiput.wikiput(self.url_prefix + pagename,
                             open(something.filename, "r").read(),
                             editsummary, username)
         elif something.__class__.__name__ == "Folder":
             if pagename is not None:
                 raise ValueError("We don't do pagenames, when we're"
                                  " sending a folder to the wiki. I"
                                  " am thinking about making it prefix"
                                  " all the page names with the"
                                  " pagename, by way of a \"prefix\""
                                  " argument, but haven't had a need"
                                  " yet for it.")
             send_everything_in_folder(something, self,
                                          username=username,
                                          editsummary=editsummary)
         elif something.__class__.__name__ == "str":
             if pagename is None:
                 pagename = "SandBox"
             else:
                 make_sure_its_camelcase(pagename)
             wikiput.wikiput(self.url_prefix + pagename,
                             something, editsummary, username)
         else:
             raise ValueError("I don't know how to receive %s." \
                              % something)
 
 
 class OddMuseWikiPage:
     
     """Represent a particular page on an OddMuse wiki.
     
     DOC
     
     DOC
     """
     
     def __init__(self, wiki, pagename):
         """Initialize the page with it's wiki, pagename."""
         self.wiki = wiki
         self.pagename = pagename
         self.raw_url = self.wiki.url_prefix + ("?action=browse;" \
                                                "id=%s;raw=1" %
                                                self.pagename)
     
     def receive(self, something, **otherstuff):
         """Place a something onto the OddMuseWiki page."""
         args = otherstuff.copy()
         args["pagename"] = self.pagename
         OddMuseWiki.receive(self.wiki, something, **args)
 
     def raw_content(self, default=None, raise_exception=True):
         """Return the raw content of the wiki page."""
         try:
             return urllib2.urlopen(self.raw_url).read()
         except Exception, e:
             if raise_exception:
                 raise e
             return default
 
 
 class RssFeed:
 
     """An RSS feed that you can check for changes.
 
     Keep "seen" and "last loaded" lists, on disk, representing the RSS
     feed. The versions on disk are pickles of entry dictionaries. The
     entry dictionaries are made from feedparser responses.
 
     When entries are loaded into memory, they are sorted oldest to
     newest.
 
     Entries can be returned one by one, or they can be returned en
     masse.
 
     DOC
     """
     
     def __init__(self, url, limiter_in_minutes=60):
         """DOC"""
         self.url = url
         self.limiter_in_minutes = limiter_in_minutes
         self.folder = folder_associated_with_url(url, prefix="RSS",
                                                  create=True)
     
     def clean_out(self):
         """Erase all memory."""
         del self.folder["lastload.pck"]
         del self.folder["seen.pck"]
         del self.folder["timestmp.pck"]
     
     def read_from_internet(self,
                            disconnected_ok=False,
                            raise_exception=True):
         """Load RSS feed entries from the web, and cache them.
 
         This can't yet distinguish whether an exception is just because
         we're not connected to the Internet, or whether it's some other
         exception.
         
         So, all exceptions are interpreted as "disconnected." If
         disconnected_ok is True, then no exception is ever raised,
         presently. The exception is urllib2.URLError.
 
         disconnected_ok trumps over raise_exception.
         
         TODO: Make this function properly figure out if the problem was
         "disconnected from the web," or if it was a deeper problem,
         possibly that the feed isn't there any more, or something.
         
         Return the list of entries. Entries are feedparser entries. They
         are returned from oldest to newest.
         """
         # Respect the limiter!
         now = time.mktime(time.gmtime())
         lasttime = self.folder.get("timestmp.pck", default=None)
         if lasttime is not None:
             lasttime = pickle.loads(lasttime)  # Turn into number.
             if (now - lasttime) < (self.limiter_in_minutes*60):
                 return []  # Nope! It's been to recent.
         # Pull the feed!
         feed = feedparser.parse(self.url)
         entries = feed.entries
         # What if it wasn't there?
         if feed.bozo == 1 and (raise_exception
                                and not disconnected_ok):
             raise feed.bozo_exception
         # Sort them, oldest to newest!
         def date_for_entry(entry):
             for key in ["updated_parsed", "published_parsed",
                         "created_parsed"]:
                 if entry.has_key(key):
                     return entry[key]
             return None
         entries.sort(lambda a,b: cmp(date_for_entry(a),
                                      date_for_entry(b)))
         # Stash them away, as dictionaries.
         for_storage = [dict(x) for x in entries]
         self.folder["lastload.pck"] = pickle.dumps(for_storage)
         # Remember UTC time, for the limiter.
         now = time.mktime(time.gmtime())
         self.folder["timestmp.pck"] = pickle.dumps(now)
         return for_storage
     
     def get_seen_list(self):
         """DOC"""
         text = self.folder.get("seen.pck",
                                "(l.")  # Empty list
         return pickle.loads(text)
     
     def get_last_loaded_list(self):
         """DOC"""
         text = self.folder.get("lastload.pck",
                                   "(l.")  # Empty list
         return pickle.loads(text)
     
     def set_seen_list(self, seen):
         """DOC"""
         self.folder["seen.pck"] = pickle.dumps(seen)
 
     def has_new_entries(self, reload=False, reload_if_done=True):
         """Return whether or not there are any new entries to read.
 
         Open up the loaded and seen lists. Do a quick compare: Anything
         new? If so, say "YES!"
 
         If we make it to the end, and we're not doing a reload, then
         report: "NO!"
 
         If we're doing the reload, then just do it, and the second time,
         don't reload at the end.
         """
         reloaded = False
         if reload:
             self.read_from_internet(raise_exception=False)
             reload = True
         seen = self.get_seen_list()
         loaded = self.get_last_loaded_list()
         for entry in loaded:
             if entry not in seen:
                 return True
         if reload_if_done and not reloaded:
             return self.has_new_entries(reload=True,
                                         reload_if_done=False)
         return False
 
     def get_all_new_entries(self, reload=False, reload_if_done=True):
         """Retrieve a list of all unseen entries, oldest to newest.
         
         Read the "seen" and "last loaded" files. For each item on the
         "last loaded" list, check if it has been seen or not. If it
         hasn't been seen before, add it to the return list.
 
         Before returning, set "seen" to be equivalent to "loaded." That
         is, we've now seen everything.
 
         If you want to force a reload, pass "reload=True." If it can't
         contact the Internet, it won't raise a fuss, though.
         
         If you want to reload only if there are no more entries left,
         pass "reload_if_done=True." This too, won't raise a fuss, if it
         doesn't connect.
         """
         reloaded = False
         if reload:
             self.read_from_internet(raise_exception=False)
             reloaded = True
         seen = self.get_seen_list()
         loaded = self.get_last_loaded_list()
         to_return = []
         # Check "loaded" list for everything not yet in the seen list.
         for entry in loaded:
             if entry not in seen:
                 to_return.append(entry)
                 seen.append(entry)
         # Handle case where we reload if we've seen everything.
         if len(to_return) == 0 and reload_if_done and not reloaded:
             return self.get_all_new_entries(reload=True,
                                             reload_if_done=False)
         self.set_seen_list(loaded)
         return [brand_dictionary(x, self, ignore_none=False)
                 for x in to_return]
     
     def get_one_new_entry(self, reload=False, reload_if_done=True):
         """Get a single new entry, the next that hasn't been seen.
 
         We take advantage of the fact that entries, upon loading, are
         ordered oldest to newist.
 
         We walk through, starting at the beginning, looking for the one
         we haven't seen before. When we find it, we note it, and then
         send it back.
 
         If we make it all the way through the list, and we've seen them
         all, we may perform a reload.
         """
         reloaded = False
         if reload:
             self.read_from_internet(raise_exception=False)
             reload = True
         seen = self.get_seen_list()
         loaded = self.get_last_loaded_list()
         to_return = None
         dirty = False
         for entry in seen:
             if entry not in loaded:
                 seen.remove(entry)
                 dirty = True
         for entry in loaded:
             if entry not in seen:
                 seen.append(entry)
                 dirty = True
                 to_return = entry
                 break
         if to_return is None and reload_if_done and not reloaded:
             return self.get_one_new_entry(reload=True,
                                           reload_if_done=False)
         if dirty:
             self.set_seen_list(seen)
         return brand_dictionary(to_return, self, ignore_none=True)
 
 
 def setup_memory_system(memories_database=None):
     """Setup the data storage system.
 
     Presently, it just makes sure that the memory_store folder exists.
     """
     if not os.path.exists(memory_store):
         os.mkdir(memory_store)
 
 
 def when(test, do_this):
     """When test is not None, send result to do_this."""
     result = test()
     while test is not None:
         do_this(result)
         result = test()
 
 
 community_wiki = OddMuseWiki("http://www.communitywiki.org/")
 community_wiki.default_username = "LionKimbro"  # Please override..!
 
 
 cw_backup_folder = Folder("C:\\cw_backups", create=True)
 
 
 def tally():
     """DOC
 
     DOC
     """
     when(community_wiki.get_one_new_entry,
          cw_backup_folder.receive)
 

Languages: