beginning of docx/pdf scrapers
Former-commit-id: 72b18d2f2bae7cfce33fb8639ad1523c7bbcc0a3
--- a/admin/refreshDesignDoc.php
+++ b/admin/refreshDesignDoc.php
@@ -9,6 +9,7 @@
$obj->language = "javascript";
$obj->views->all->map = "function(doc) { emit(doc._id, doc); };";
$obj->views->byDate->map = "function(doc) { emit(doc.date, doc); };";
+$obj->views->byDate->reduce = "_count";
$obj->views->byDateMonthYear->map = "function(doc) { emit(doc.date, doc); };";
$obj->views->byDateMonthYear->reduce = "_count";
$obj->views->byAgencyID->map = "function(doc) { emit(doc.agencyID, doc); };";
--- a/documents/about.php
+++ /dev/null
@@ -1,11 +1,1 @@
-<?php
-include('template.inc.php');
-include_header_documents("");
-include_once('../include/common.inc.php');
-?>
-<h1>About</h1>
-<?php
-include_footer_documents();
-?>
-
--- a/documents/genericScrapers.py
+++ b/documents/genericScrapers.py
@@ -11,6 +11,19 @@
from datetime import *
import codecs
+from StringIO import StringIO
+
+from docx import *
+from lxml import etree
+import zipfile
+
+from pdfminer.pdfparser import PDFDocument, PDFParser
+from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter, process_pdf
+from pdfminer.pdfdevice import PDFDevice, TagExtractor
+from pdfminer.converter import TextConverter
+from pdfminer.cmapdb import CMapDB
+from pdfminer.layout import LAParams
+
class GenericDisclogScraper(object):
__metaclass__ = abc.ABCMeta
agencyID = None
@@ -35,11 +48,78 @@
""" do the scraping """
return
- @abc.abstractmethod
- def getDescription(self, content, entry, doc):
- """ get description"""
- return
-
+class GenericPDFDisclogScraper(GenericDisclogScraper):
+
+ def doScrape(self):
+ foidocsdb = scrape.couch['disclosr-foidocuments']
+ (url,mime_type,content) = scrape.fetchURL(scrape.docsdb, self.getURL(), "foidocuments", self.getAgencyID())
+
+ laparams = LAParams()
+
+ rsrcmgr = PDFResourceManager(caching=True)
+
+ outfp = StringIO.StringIO()
+
+ device = TextConverter(rsrcmgr, outfp, codec='utf-8', laparams=laparams)
+
+
+ fp = StringIO.StringIO()
+ fp.write(content)
+ description = output.getvalue();
+ process_pdf(rsrcmgr, device, fp, set(), caching=True, check_extractable=True)
+ fp.close()
+ device.close()
+ outfp.close()
+
+ hash = scrape.mkhash(description)
+ #print hash
+ doc = foidocsdb.get(hash)
+ #print doc
+ if doc == None:
+ print "saving "+ hash
+ edate = datetime.fromtimestamp(mktime( )).strftime("%Y-%m-%d")
+ doc = {'_id': hash, 'agencyID': self.getAgencyID(), 'url': self.getURL(), 'docID': hash,
+ "date": edate,"title": "Disclosure Log Updated"}
+ self.getDescription(entry,entry, doc)
+ foidocsdb.save(doc)
+ else:
+ print "already saved"
+
+
+class GenericDOCXDisclogScraper(GenericDisclogScraper):
+
+ def doScrape(self):
+ foidocsdb = scrape.couch['disclosr-foidocuments']
+ (url,mime_type,content) = scrape.fetchURL(scrape.docsdb, self.getURL(), "foidocuments", self.getAgencyID())
+
+ mydoc = zipfile.ZipFile(file)
+ xmlcontent = mydoc.read('word/document.xml')
+ document = etree.fromstring(xmlcontent)
+
+ ## Fetch all the text out of the document we just created
+ paratextlist = getdocumenttext(document)
+
+ # Make explicit unicode version
+ newparatextlist = []
+ for paratext in paratextlist:
+ newparatextlist.append(paratext.encode("utf-8"))
+
+ ## Print our documnts test with two newlines under each paragraph
+ description = '\n\n'.join(newparatextlist)
+
+ hash = scrape.mkhash(description)
+ #print hash
+ doc = foidocsdb.get(hash)
+ #print doc
+ if doc == None:
+ print "saving "+ hash
+ edate = datetime.fromtimestamp(mktime()).strftime("%Y-%m-%d")
+ doc = {'_id': hash, 'agencyID': self.getAgencyID(), 'url': self.getURL(), 'docID': hash,
+ "date": edate,"title": "Disclosure Log Updated"}
+ self.getDescription(entry,entry, doc)
+ foidocsdb.save(doc)
+ else:
+ print "already saved"
class GenericRSSDisclogScraper(GenericDisclogScraper):
@@ -137,12 +217,7 @@
self.getDescription(description,row, doc)
if notes != None:
doc.update({ 'notes': (''.join(notes.stripped_strings))})
- badtitles = ['-','Summary of FOI Request','FOI request(in summary form)','Summary of FOI request received by the ASC',
-'Summary of FOI request received by agency/minister','Description of Documents Requested','FOI request','Description of FOI Request','Summary of request','Description','Summary',
-'Summary of FOIrequest received by agency/minister','Summary of FOI request received','Description of FOI Request',"FOI request",'Results 1 to 67 of 67']
- if doc['title'] not in badtitles and doc['description'] != '':
- print "saving"
- foidocsdb.save(doc)
+ foidocsdb.save(doc)
else:
print "already saved "+hash
--- a/documents/rss.xml.php
+++ b/documents/rss.xml.php
@@ -9,12 +9,11 @@
$TestFeed = new RSS2FeedWriter();
//Setting the channel elements
//Use wrapper functions for common channelelements
-$TestFeed->setTitle('Last Modified - All');
+$TestFeed->setTitle('disclosurelo.gs Newest Entries - All');
$TestFeed->setLink('http://disclosurelo.gs/rss.xml.php');
-$TestFeed->setDescription('Latest entries');
+$TestFeed->setDescription('disclosurelo.gs Newest Entries - All Agencies');
$TestFeed->setChannelElement('language', 'en-us');
$TestFeed->setChannelElement('pubDate', date(DATE_RSS, time()));
-
//Retriving informations from database
$idtoname = Array();
$agenciesdb = $server->get_db('disclosr-agencies');
@@ -29,11 +28,10 @@
$newItem = $TestFeed->createNewItem();
//Add elements to the feed item
$newItem->setTitle($row->value->title);
- $newItem->setLink("view.php?id=".$row->value->_id);
- $newItem->setDate(date("c", strtotime($row->value->date)));
+ $newItem->setLink("http://disclosurelo.gs/view.php?id=".$row->value->_id);
+ $newItem->setDate(strtotime($row->value->date));
$newItem->setDescription(displayLogEntry($row,$idtoname));
- $newItem->setAuthor($idtoname[$row->value->agencyID]);
- $newItem->addElement('guid', $row->value->_id,array('isPermaLink'=>'true'));
+ $newItem->addElement('guid', "http://disclosurelo.gs/view.php?id=".$row->value->_id,array('isPermaLink'=>'true'));
//Now add the feed item
$TestFeed->addItem($newItem);
}
--- a/documents/scrapers/227cb6eb7d2c9f8a6e846df7447d6caa.py
+++ b/documents/scrapers/227cb6eb7d2c9f8a6e846df7447d6caa.py
@@ -21,10 +21,9 @@
for row in soup.find(class_ = "ms-rteTable-GreyAlternating").find_all('tr'):
if row != None:
rowtitle = row.find('th').string
- if rowtitle != None:
- description = description + "\n" + rowtitle + ": "
+ description = description + "\n" + rowtitle + ": "
for text in row.find('td').stripped_strings:
- description = description + text
+ description = description + text
for atag in row.find_all("a"):
if atag.has_key('href'):
links.append(scrape.fullurl(link,atag['href']))
--- a/documents/template.inc.php
+++ b/documents/template.inc.php
@@ -1,169 +1,152 @@
<?php
function include_header_documents($title) {
- header('X-UA-Compatible: IE=edge,chrome=1');
- ?>
- <!doctype html>
- <!-- paulirish.com/2008/conditional-stylesheets-vs-css-hacks-answer-neither/ -->
- <!--[if lt IE 7]> <html class="no-js lt-ie9 lt-ie8 lt-ie7" lang="en"> <![endif]-->
- <!--[if IE 7]> <html class="no-js lt-ie9 lt-ie8" lang="en"> <![endif]-->
- <!--[if IE 8]> <html class="no-js lt-ie9" lang="en"> <![endif]-->
- <!-- Consider adding a manifest.appcache: h5bp.com/d/Offline -->
- <!--[if gt IE 8]><!--> <html class="no-js" lang="en"> <!--<![endif]-->
- <head>
- <meta charset="utf-8">
+?>
+<!doctype html>
+<!-- paulirish.com/2008/conditional-stylesheets-vs-css-hacks-answer-neither/ -->
+<!--[if lt IE 7]> <html class="no-js lt-ie9 lt-ie8 lt-ie7" lang="en"> <![endif]-->
+<!--[if IE 7]> <html class="no-js lt-ie9 lt-ie8" lang="en"> <![endif]-->
+<!--[if IE 8]> <html class="no-js lt-ie9" lang="en"> <![endif]-->
+<!-- Consider adding a manifest.appcache: h5bp.com/d/Offline -->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en"> <!--<![endif]-->
+<head>
+ <meta charset="utf-8">
- <title>Australian Disclosure Logs<?php if ($title != "") echo " - $title"; ?></title>
- <meta name="description" content="">
+ <!-- Use the .htaccess and remove these lines to avoid edge case issues.
+ More info: h5bp.com/i/378 -->
+ <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
- <!-- Mobile viewport optimized: h5bp.com/viewport -->
- <meta name="viewport" content="width=device-width">
- <link rel="alternate" type="application/rss+xml" title="Latest Disclosure Log Entries" href="rss.xml.php" />
- <!-- Place favicon.ico and apple-touch-icon.png in the root directory: mathiasbynens.be/notes/touch-icons -->
- <meta name="google-site-verification" content="jkknX5g2FCpQvrW030b1Nq2hyoa6mb3EDiA7kCoHNj8" />
+ <title>Australian Disclosure Logs<?php if ($title != "") echo " - $title";?></title>
+ <meta name="description" content="">
- <!-- Le styles -->
- <link href="css/bootstrap.min.css" rel="stylesheet">
- <style type="text/css">
- body {
- padding-top: 60px;
- padding-bottom: 40px;
- }
- .sidebar-nav {
- padding: 9px 0;
- }
- </style>
- <link href="css/bootstrap-responsive.min.css" rel="stylesheet">
+ <!-- Mobile viewport optimized: h5bp.com/viewport -->
+ <meta name="viewport" content="width=device-width">
+<link rel="alternate" type="application/rss+xml" title="Latest Disclosure Log Entries" href="rss.xml.php" />
+ <!-- Place favicon.ico and apple-touch-icon.png in the root directory: mathiasbynens.be/notes/touch-icons -->
+<meta name="google-site-verification" content="jkknX5g2FCpQvrW030b1Nq2hyoa6mb3EDiA7kCoHNj8" />
- <!-- HTML5 shim, for IE6-8 support of HTML5 elements -->
- <!--[if lt IE 9]>
- <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
- <![endif]-->
- <!-- More ideas for your <head> here: h5bp.com/d/head-Tips -->
+ <!-- Le styles -->
+ <link href="css/bootstrap.min.css" rel="stylesheet">
+ <style type="text/css">
+ body {
+ padding-top: 60px;
+ padding-bottom: 40px;
+ }
+ .sidebar-nav {
+ padding: 9px 0;
+ }
+ </style>
+ <link href="css/bootstrap-responsive.min.css" rel="stylesheet">
- <!-- All JavaScript at the bottom, except this Modernizr build.
- Modernizr enables HTML5 elements & feature detects for optimal performance.
- Create your own custom Modernizr build: www.modernizr.com/download/
- <script src="js/libs/modernizr-2.5.3.min.js"></script>-->
- <script src="js/jquery.js"></script>
- <script type="text/javascript" src="js/flotr2.min.js"></script>
+ <!-- HTML5 shim, for IE6-8 support of HTML5 elements -->
+ <!--[if lt IE 9]>
+ <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
+ <![endif]-->
+ <!-- More ideas for your <head> here: h5bp.com/d/head-Tips -->
- </head>
- <body>
- <div class="navbar navbar-inverse navbar-fixed-top">
- <div class="navbar-inner">
- <div class="container-fluid">
- <a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </a>
- <a class="brand" href="#">Australian Disclosure Logs</a>
- <div class="nav-collapse collapse">
- <p class="navbar-text pull-right">
- Check out our subsites on:
- <a href="http://orgs.disclosurelo.gs">Government Agencies</a>
- • <a href="http://lobbyists.disclosurelo.gs">Political Lobbyists</a>
- • <a href="http://contracts.disclosurelo.gs">Government Contracts and Spending</a>
+ <!-- All JavaScript at the bottom, except this Modernizr build.
+ Modernizr enables HTML5 elements & feature detects for optimal performance.
+ Create your own custom Modernizr build: www.modernizr.com/download/
+ <script src="js/libs/modernizr-2.5.3.min.js"></script>-->
+ <script src="js/jquery.js"></script>
+ <script type="text/javascript" src="js/flotr2.min.js"></script>
+
+</head>
+<body>
+ <div class="navbar navbar-inverse navbar-fixed-top">
+ <div class="navbar-inner">
+ <div class="container-fluid">
+ <a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </a>
+ <a class="brand" href="#">Australian Disclosure Logs</a>
+ <div class="nav-collapse collapse">
+ <p class="navbar-text pull-right">
+ Check out our subsites on:
+<a href="http://orgs.disclosurelo.gs">Government Agencies</a>
+• <a href="http://lobbyists.disclosurelo.gs">Political Lobbyists</a>
+• <a href="http://contracts.disclosurelo.gs">Government Contracts and Spending</a>
- </p>
- <ul class="nav">
- <li><a href="index.php">Home</a></li>
- <li><a href="disclogsList.php">List of Disclosure Logs</a></li>
- <li><a href="about.php">About</a></li>
+ </p>
+ <ul class="nav">
+ <li><a href="index.php">Home</a></li>
+ <li><a href="disclogsList.php">List of Disclosure Logs</a></li>
+ <li><a href="about.php">About</a></li>
+
+ </ul>
+ </div><!--/.nav-collapse -->
+ </div>
+ </div>
+ </div>
+ <div class="container">
+ <?php
+}
+function include_footer_documents() {
+ ?>
+ </div> <!-- /container -->
+ <hr>
- </ul>
- </div><!--/.nav-collapse -->
- </div>
- </div>
- </div>
- <div class="container">
- <?php
- }
+ <footer>
+ <p>© Company 2012</p>
+ </footer>
+ <script type="text/javascript">
- function include_footer_documents() {
- ?>
- </div> <!-- /container -->
- <hr>
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-12341040-4']);
+ _gaq.push(['_setDomainName', 'disclosurelo.gs']);
+ _gaq.push(['_setAllowLinker', true]);
+ _gaq.push(['_trackPageview']);
- <footer>
- <p>Not affiliated with or endorsed by any government agency.</p>
- </footer>
- <script type="text/javascript">
+ (function() {
+ var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+ ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+ var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+ })();
- var _gaq = _gaq || [];
- _gaq.push(['_setAccount', 'UA-12341040-4']);
- _gaq.push(['_setDomainName', 'disclosurelo.gs']);
- _gaq.push(['_setAllowLinker', true]);
- _gaq.push(['_trackPageview']);
-
- (function() {
- var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
- ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
- var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
- })();
-
- </script>
- <!-- Le javascript
- ================================================== -->
- <!-- Placed at the end of the document so the pages load faster -->
- <!--
- <script src="js/bootstrap-transition.js"></script>
- <script src="js/bootstrap-alert.js"></script>
- <script src="js/bootstrap-modal.js"></script>
- <script src="js/bootstrap-dropdown.js"></script>
- <script src="js/bootstrap-scrollspy.js"></script>
- <script src="js/bootstrap-tab.js"></script>
- <script src="js/bootstrap-tooltip.js"></script>
- <script src="js/bootstrap-popover.js"></script>
- <script src="js/bootstrap-button.js"></script>
- <script src="js/bootstrap-collapse.js"></script>
- <script src="js/bootstrap-carousel.js"></script>
- <script src="js/bootstrap-typeahead.js"></script>-->
+</script>
+ <!-- Le javascript
+ ================================================== -->
+ <!-- Placed at the end of the document so the pages load faster -->
+<!--
+ <script src="js/bootstrap-transition.js"></script>
+ <script src="js/bootstrap-alert.js"></script>
+ <script src="js/bootstrap-modal.js"></script>
+ <script src="js/bootstrap-dropdown.js"></script>
+ <script src="js/bootstrap-scrollspy.js"></script>
+ <script src="js/bootstrap-tab.js"></script>
+ <script src="js/bootstrap-tooltip.js"></script>
+ <script src="js/bootstrap-popover.js"></script>
+ <script src="js/bootstrap-button.js"></script>
+ <script src="js/bootstrap-collapse.js"></script>
+ <script src="js/bootstrap-carousel.js"></script>
+ <script src="js/bootstrap-typeahead.js"></script>-->
- </body>
- </html>
- <?php
-}
-
-function truncate($string, $length, $stopanywhere = false) {
- //truncates a string to a certain char length, stopping on a word if not specified otherwise.
- if (strlen($string) > $length) {
- //limit hit!
- $string = substr($string, 0, ($length - 3));
- if ($stopanywhere) {
- //stop anywhere
- $string .= '...';
- } else {
- //stop on a word.
- $string = substr($string, 0, strrpos($string, ' ')) . '...';
- }
- }
- return $string;
+ </body>
+</html>
+<?php
}
function displayLogEntry($row, $idtoname) {
$result = "";
- $result .= '<div itemscope itemtype="http://schema.org/Article">';
- $result .= '<h2> <span itemprop="datePublished">' . $row->value->date . "</span>: <span itemprop='name headline'>" . truncate($row->value->title, 120)."</span>";
- $result .= '(<span itemprop="author publisher creator">' . $idtoname[$row->value->agencyID] . '</span>)</h2>';
- $result .= "<p itemprop='description articleBody text'> Title" . $row->value->title . "<br/>" . str_replace("\n", "<br>", $row->value->description);
- if (isset($row->value->notes)) {
- $result .= " <br>Note: " . $row->value->notes;
- }
- $result .= "</p>";
+ $result .= "<div><h2>".$row->value->date.": ".$row->value->title." (".$idtoname[$row->value->agencyID].")</h2> <p>".str_replace("\n","<br>",$row->value->description);
+if (isset($row->value->notes)) {
+$result .= " <br>Note: ".$row->value->notes;
+}
+$result .= "</p>";
- if (isset($row->value->links)) {
- $result .= '<h3>Links/Documents</h3><ul itemprop="associatedMedia">';
- foreach ($row->value->links as $link) {
- $result .= '<li itemscope itemtype="http://schema.org/MediaObject"><a href='.$link.' itemprop="url contentURL">' . urlencode($link) . "</a></li>";
- }
+if (isset($row->value->links)){
+$result .= "<h3>Links/Documents</h3><ul>";
+foreach ($row->value->links as $link) {
+ $result .= "<li><a href='$link'>".$link."</a></li>";
+}
$result .= "</ul>";
- }
- $result .= "<small><A itemprop='url' href='" . $row->value->url . "'>View original source...</a> ID: " . strip_tags($row->value->docID) . "</small>";
- $result .= "</div>";
- return $result;
+}
+ $result .= "<small><A href='".$row->value->url."'>View original source...</a> ID: ".strip_tags($row->value->docID)."</small>";
+$result .= "</div>";
+return $result;
}
--- a/lib/FeedWriter
+++ /dev/null
--- /dev/null
+++ b/lib/FeedWriter/COPYING
@@ -1,1 +1,675 @@
-
+ GNU GENERAL PUBLIC LICENSE
+ Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+ The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users. We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors. You can apply it to
+your programs, too.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+ To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights. Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+ For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received. You must make sure that they, too, receive
+or can get the source code. And you must show them these terms so they
+know their rights.
+
+ Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+ For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software. For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+ Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so. This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software. The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable. Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products. If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+ Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary. To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ TERMS AND CONDITIONS
+
+ 0. Definitions.
+
+ "This License" refers to version 3 of the GNU General Public License.
+
+ "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+ "The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+ To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy. The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+ A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+ To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+ To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+ An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+ 1. Source Code.
+
+ The "source code" for a work means the preferred form of the work
+for making modifications to it. "Object code" means any non-source
+form of a work.
+
+ A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+ The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+ The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+ The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+ The Corresponding Source for a work in source code form is that
+same work.
+
+ 2. Basic Permissions.
+
+ All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+ You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license othe