From: Maxious Date: Wed, 28 Nov 2012 22:49:23 +0000 Subject: more scrapers X-Git-Url: http://maxious.lambdacomplex.org/git/?p=disclosr.git&a=commitdiff&h=ac0ef2a98459aa0540a63588a8774c48ca2cca7b --- more scrapers Former-commit-id: 96bfae466ef6496e0bc9469d556f599c5e0e3d92 --- --- a/.gitmodules +++ b/.gitmodules @@ -31,4 +31,7 @@ [submodule "documents/lib/parsedatetime"] path = documents/lib/parsedatetime url = git://github.com/bear/parsedatetime.git +[submodule "lib/FeedWriter"] + path = lib/FeedWriter + url = https://github.com/mibe/FeedWriter --- /dev/null +++ b/documents/.gitignore @@ -1,1 +1,2 @@ +*.pyc --- a/documents/disclogsList.php +++ b/documents/disclogsList.php @@ -35,18 +35,18 @@ echo '
(' . 'view local copy)'; } else { - echo "✘"; + echo "✘"; } echo "\n"; if (isset($row->value->FOIDocumentsURL)) { if (file_exists("./scrapers/" . $row->id . '.py')) { - echo "✔"; + echo "✔"; $green++; } else if (file_exists("./scrapers/" . $row->id . '.txt')) { - echo "▬"; + echo "▬"; $orange++; } else { - echo "✘"; + echo "✘"; $red++; } } @@ -57,7 +57,8 @@ setteErrorHandler($e); } echo ""; -echo $agencies." agencies ".(($disclogs/$agencies)*100)."% with disclosure logs, ".(($green/$disclogs)*100)."% with scrapers ".(($red/$disclogs)*100)."% without scrapers ".(($orange/$disclogs)*100)."% WIP scrapers "; +echo $agencies." agencies, ".round(($disclogs/$agencies)*100)."% with disclosure logs; " +.round(($green/$disclogs)*100)."% logs with scrapers ".round(($red/$disclogs)*100)."% logs without scrapers ".round(($orange/$disclogs)*100)."% logs Work-In-Progress scrapers "; include_footer_documents(); ?> --- a/documents/genericScrapers.py +++ b/documents/genericScrapers.py @@ -92,7 +92,8 @@ return table.find_all('tr') def getDate(self, content, entry, doc): date = ''.join(content.stripped_strings).strip() - date = str.replace("Octber","October",date) + date = date.replace("Octber","October") + print date edate = parse(date, dayfirst=True, fuzzy=True).strftime("%Y-%m-%d") print edate doc.update({'date': edate}) --- a/documents/index.php +++ b/documents/index.php @@ -20,7 +20,7 @@ if ($rows) { foreach ($rows as $row) { -displayLogEntry($row,$idtoname); +echo displayLogEntry($row,$idtoname); } } } catch (SetteeRestClientException $e) { --- a/documents/robots.txt +++ b/documents/robots.txt @@ -2,4 +2,5 @@ # http://code.google.com/web/controlcrawlindex/ User-agent: * - +Disallow: /admin/ +Sitemap: http://disclosurelo.gs/sitemap.xml.php --- a/documents/rss.xml.php +++ b/documents/rss.xml.php @@ -3,28 +3,38 @@ // Agency X updated Y, new files, diff of plain text/link text, // feed for just one agency or all // This is a minimum example of using the Universal Feed Generator Class -include("lib/FeedWriter.php"); +include("../lib/FeedWriter/FeedTypes.php"); +include_once('../include/common.inc.php'); //Creating an instance of FeedWriter class. -$TestFeed = new FeedWriter(RSS2); +$TestFeed = new RSS2FeedWriter(); //Setting the channel elements //Use wrapper functions for common channelelements $TestFeed->setTitle('Last Modified - All'); -$TestFeed->setLink('http://disclosr.lambdacomplex.org/rss.xml.php'); +$TestFeed->setLink('http://disclosurelo.gs/rss.xml.php'); $TestFeed->setDescription('This is test of creating a RSS 2.0 feed Universal Feed Writer'); + $TestFeed->setChannelElement('language', 'en-us'); + $TestFeed->setChannelElement('pubDate', date(DATE_RSS, time())); //Retriving informations from database -$rows = $db->get_view("app", "byLastModified")->rows; +$idtoname = Array(); +$agenciesdb = $server->get_db('disclosr-agencies'); +foreach ($agenciesdb->get_view("app", "byCanonicalName")->rows as $row) { + $idtoname[$row->id] = trim($row->value->name); +} +$foidocsdb = $server->get_db('disclosr-foidocuments'); +$rows = $foidocsdb->get_view("app", "byDate", Array('9999-99-99','0000-00-00'), true)->rows; //print_r($rows); foreach ($rows as $row) { //Create an empty FeedItem $newItem = $TestFeed->createNewItem(); //Add elements to the feed item - $newItem->setTitle($row['name']); - $newItem->setLink($row['id']); - $newItem->setDate(date("c", $row['metadata']['lastModified'])); - $newItem->setDescription($row['name']); + $newItem->setTitle($row->value->title); + $newItem->setLink("view.php?id=".$row->value->docID); + $newItem->setDate(date("c", strtotime($row->value->date))); + $newItem->setDescription(displayLogEntry($row,$idtoname)); + $newItem->addElement('guid', $row->value->_id,array('isPermaLink'=>'true')); //Now add the feed item $TestFeed->addItem($newItem); } //OK. Everything is done. Now genarate the feed. -$TestFeed->genarateFeed(); +$TestFeed->generateFeed(); ?> --- /dev/null +++ b/documents/scrapers/00a294de663db69062ca09aede7c0487.txt @@ -1,1 +1,2 @@ +multipage --- /dev/null +++ b/documents/scrapers/1803322b27286950cab0c543168b5f21.txt @@ -1,1 +1,2 @@ +multipage log --- /dev/null +++ b/documents/scrapers/24bd71114d3975ed9a63ad29624c62c9.py @@ -1,1 +1,21 @@ +import sys,os +sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../')) +import genericScrapers +import scrape +from bs4 import BeautifulSoup +#http://www.doughellmann.com/PyMOTW/abc/ +class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper): + def getTable(self,soup): + return soup.find(id = "inner_content") + def getColumnCount(self): + return 2 + def getColumns(self,columns): + (date, title) = columns + return (date, date, title, title, None) + +if __name__ == '__main__': + print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper) + print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper) + ScraperImplementation().doScrape() + --- /dev/null +++ b/documents/scrapers/31685505438d393f45a90f442b8fa27f.txt @@ -1,1 +1,2 @@ +pdf --- /dev/null +++ b/documents/scrapers/41a166419503bb50e410c58be54c102f.txt @@ -1,1 +1,1 @@ - +aspx --- /dev/null +++ b/documents/scrapers/4934000fddd6a5b1094f398798341290.py @@ -1,1 +1,23 @@ +import sys,os +sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../')) +import genericScrapers +import scrape +from bs4 import BeautifulSoup +import dateutil +from dateutil.parser import * +from datetime import * +#http://www.doughellmann.com/PyMOTW/abc/ +class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper): + def getColumnCount(self): + return 5 + def getColumns(self,columns): + (id, date, title, description, notes) = columns + return (id, date, title, description, notes) + +if __name__ == '__main__': + print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper) + print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper) + si = ScraperImplementation() + si.doScrape() + --- /dev/null +++ b/documents/scrapers/601aedeef4344638d635bdd761e9fdba.py @@ -1,1 +1,21 @@ +import sys,os +sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../')) +import genericScrapers +import scrape +from bs4 import BeautifulSoup +#http://www.doughellmann.com/PyMOTW/abc/ +class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper): + #def getTable(self,soup): + # return soup.find(id = "ctl00_PlaceHolderMain_intro2__ControlWrapper_CerRichHtmlField").table + def getColumnCount(self): + return 4 + def getColumns(self,columns): + (date, title, description,notes) = columns + return (title, date, title, description, notes) + +if __name__ == '__main__': + print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper) + print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper) + ScraperImplementation().doScrape() + --- /dev/null +++ b/documents/scrapers/649b053f5e2884906ddc7174c2cd4b38.py @@ -1,1 +1,28 @@ +import sys,os +sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../')) +import genericScrapers +import scrape +from bs4 import BeautifulSoup +import dateutil +from dateutil.parser import * +from datetime import * +#http://www.doughellmann.com/PyMOTW/abc/ +class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper): + def getColumnCount(self): + return 5 + def getColumns(self,columns): + (id, date, title, description, notes) = columns + return (id, date, title, description, notes) + +if __name__ == '__main__': + print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper) + print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper) + si = ScraperImplementation() + si.doScrape() + si.disclogURL = "http://www.fahcsia.gov.au/disclosure-log-2011-12-financial-year" + si.doScrape() + si.disclogURL = "http://www.fahcsia.gov.au/disclosure-log-2010-11-financial-year" + si.doScrape() + + --- /dev/null +++ b/documents/scrapers/6afdde1d4ff1ad8d8cfe1a8675ea83bd.txt @@ -1,1 +1,2 @@ +PDF --- /dev/null +++ b/documents/scrapers/6cf3870aedeeecfd6394b5c0abed4c55.py @@ -1,1 +1,23 @@ +import sys,os +sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../')) +import genericScrapers +import scrape +from bs4 import BeautifulSoup +import dateutil +from dateutil.parser import * +from datetime import * +#http://www.doughellmann.com/PyMOTW/abc/ +class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper): + def getColumnCount(self): + return 5 + def getColumns(self,columns): + (id, date, title, description, notes) = columns + return (id, date, title, description, notes) + +if __name__ == '__main__': + print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper) + print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper) + si = ScraperImplementation() + si.doScrape() + --- /dev/null +++ b/documents/scrapers/72a295f10734d64e8185f651fd2b39ea.txt @@ -1,1 +1,2 @@ +weird div based log with tables of links --- /dev/null +++ b/documents/scrapers/75d8f1c605ef9da0c2590264b7aa046b.py @@ -1,1 +1,21 @@ +import sys,os +sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../')) +import genericScrapers +import scrape +from bs4 import BeautifulSoup +#http://www.doughellmann.com/PyMOTW/abc/ +class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper): + def getTable(self,soup): + return soup.find(id = "content-middle").table + def getColumnCount(self): + return 5 + def getColumns(self,columns): + (id, date, title, description,notes) = columns + return (id, date, title, description, notes) + +if __name__ == '__main__': + print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper) + print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper) + ScraperImplementation().doScrape() + --- /dev/null +++ b/documents/scrapers/8796220032faf94501bd366763263685.txt @@ -1,1 +1,2 @@ +multiple pages --- /dev/null +++ b/documents/scrapers/8e874a2fde8aa0ccdc6d14573d766540.txt @@ -1,1 +1,49 @@ +import sys,os +sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../')) +import genericScrapers +import scrape +from bs4 import BeautifulSoup +#http://www.doughellmann.com/PyMOTW/abc/ +class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper): + def getDescription(self,content, entry,doc): + link = None + links = [] + description = "" + for atag in entry.find_all('a'): + if atag.has_key('href'): + link = scrape.fullurl(self.getURL(),atag['href']) + (url,mime_type,htcontent) = scrape.fetchURL(scrape.docsdb, link, "foidocuments", self.getAgencyID(), False) + if htcontent != None: + if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml": + # http://www.crummy.com/software/BeautifulSoup/documentation.html + soup = BeautifulSoup(htcontent) + for row in soup.find(class_ = "ms-rteTable-GreyAlternating").find_all('tr'): + if row != None: + rowtitle = row.find('th').string + description = description + "\n" + rowtitle + ": " + for text in row.find('td').stripped_strings: + description = description + text + for atag in row.find_all("a"): + if atag.has_key('href'): + links.append(scrape.fullurl(link,atag['href'])) + + if links != []: + doc.update({'links': links}) + if description != "": + doc.update({ 'description': description}) + + def getColumnCount(self): + return 2 + def getTable(self,soup): + return soup.find(class_ = "ms-rteTable-GreyAlternating") + def getColumns(self,columns): + (date, title) = columns + return (title, date, title, title, None) + +if __name__ == '__main__': + print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper) + print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper) + ScraperImplementation().doScrape() +# old site too http://archive.treasury.gov.au/content/foi_publications.asp + --- /dev/null +++ b/documents/scrapers/905a1c409b6afb1de0074b13a5559560.py @@ -1,1 +1,23 @@ +import sys,os +sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../')) +import genericScrapers +import scrape +from bs4 import BeautifulSoup +import dateutil +from dateutil.parser import * +from datetime import * +#http://www.doughellmann.com/PyMOTW/abc/ +class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper): + def getColumnCount(self): + return 5 + def getColumns(self,columns): + (id, date, title, description, notes) = columns + return (id, date, title, description, notes) + +if __name__ == '__main__': + print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper) + print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper) + si = ScraperImplementation() + si.doScrape() + --- /dev/null +++ b/documents/scrapers/b0a3281ba66efe173c5a33d5ef90ff76.txt @@ -1,1 +1,2 @@ +multipage immi --- /dev/null +++ b/documents/scrapers/b0fb402314e685238537105ee0e70c84.py @@ -1,1 +1,21 @@ +import sys,os +sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../')) +import genericScrapers +import scrape +from bs4 import BeautifulSoup +#http://www.doughellmann.com/PyMOTW/abc/ +class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper): + #def getTable(self,soup): + # return soup.find(id = "ctl00_PlaceHolderMain_intro2__ControlWrapper_CerRichHtmlField").table + def getColumnCount(self): + return 5 + def getColumns(self,columns): + (id, date, title, description,notes) = columns + return (id, date, title, description, notes) + +if __name__ == '__main__': + print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper) + print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper) + ScraperImplementation().doScrape() + --- /dev/null +++ b/documents/scrapers/b7770c4584332cff42bb6abb3326e564.py @@ -1,1 +1,21 @@ +import sys,os +sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../')) +import genericScrapers +import scrape +from bs4 import BeautifulSoup +#http://www.doughellmann.com/PyMOTW/abc/ +class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper): + def getTable(self,soup): + return soup.find(id = "ctl00_PlaceHolderMain_Content__ControlWrapper_RichHtmlField").table + def getColumnCount(self): + return 5 + def getColumns(self,columns): + (id, date, title, description,notes) = columns + return (id, date, title, description, notes) + +if __name__ == '__main__': + print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper) + print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper) + ScraperImplementation().doScrape() + --- /dev/null +++ b/documents/scrapers/e90b1b7cbb83e3eed0b5f849c7e3af79.py @@ -1,1 +1,21 @@ +import sys,os +sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../')) +import genericScrapers +import scrape +from bs4 import BeautifulSoup +#http://www.doughellmann.com/PyMOTW/abc/ +class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper): + def getTable(self,soup): + return soup.find(id = "inner_content") + def getColumnCount(self): + return 2 + def getColumns(self,columns): + (date, title) = columns + return (date, date, title, title, None) + +if __name__ == '__main__': + print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper) + print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper) + ScraperImplementation().doScrape() + --- /dev/null +++ b/documents/scrapers/f5ce2d1651739704634eb8ca4b2b46d3.py @@ -1,1 +1,21 @@ +import sys,os +sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../')) +import genericScrapers +import scrape +from bs4 import BeautifulSoup +#http://www.doughellmann.com/PyMOTW/abc/ +class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper): + def getTable(self,soup): + return soup.find(id = "ctl00_PlaceHolderMain_PublishingPageContent__ControlWrapper_RichHtmlField").table + def getColumnCount(self): + return 7 + def getColumns(self,columns): + (id, date, title, description,link,deldate, notes) = columns + return (id, date, title, description, notes) + +if __name__ == '__main__': + print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper) + print 'Instance:', isinstance(ScraperImplementation(), genericScrapers.GenericOAICDisclogScraper) + ScraperImplementation().doScrape() + --- /dev/null +++ b/documents/sitemap.xml.php @@ -1,1 +1,25 @@ +"; +echo '' . "\n"; +echo " " . local_url() . "index.php1.0\n"; +foreach (scandir("./") as $file) { + if (strpos($file, ".php") !== false && $file != "index.php" && $file != "sitemap.xml.php") + echo " " . local_url() . "$file0.6\n"; +} + +$db = $server->get_db('disclosr-foidocuments'); +try { + $rows = $db->get_view("app", "all")->rows; + foreach ($rows as $row) { + echo '' . local_url() . 'view.php?id=' . $row->value->_id . "0.3\n"; + } +} catch (SetteeRestClientException $e) { + setteErrorHandler($e); +} +echo ''; +?> + --- a/documents/template.inc.php +++ b/documents/template.inc.php @@ -130,21 +130,23 @@ } function displayLogEntry($row, $idtoname) { - echo "

".$row->value->date.": ".$row->value->title." (".$idtoname[$row->value->agencyID].")

".str_replace("\n","
",$row->value->description); + $result = ""; + $result .= "

".$row->value->date.": ".$row->value->title." (".$idtoname[$row->value->agencyID].")

".str_replace("\n","
",$row->value->description); if (isset($row->value->notes)) { -echo "
Note: ".$row->value->notes; +$result .= "
Note: ".$row->value->notes; } -echo "

"; +$result .= "

"; if (isset($row->value->links)){ -echo "

Links/Documents

    "; +$result .= "

    Links/Documents

      "; foreach ($row->value->links as $link) { - echo "
    • ".$link."
      • "; + $result .= "
    • ".$link."
      • "; } - echo "
    "; + $result .= "
"; } - echo "View original source... ID: ".$row->value->docID.""; -echo"
"; + $result .= "View original source... ID: ".$row->value->docID.""; +$result .= "
"; +return $result; } --- /dev/null +++ b/documents/view.php @@ -1,1 +1,27 @@ + +get_db('disclosr-agencies'); + +$idtoname = Array(); +foreach ($agenciesdb->get_view("app", "byCanonicalName")->rows as $row) { + $idtoname[$row->id] = trim($row->value->name); +} +$foidocsdb = $server->get_db('disclosr-foidocuments'); +try { + $obj = new stdClass(); + $obj->value = $foidocsdb->get($_REQUEST['id']); +echo displayLogEntry($obj,$idtoname); + +} catch (SetteeRestClientException $e) { + setteErrorHandler($e); +} +include_footer_documents(); +?> + --- a/lib/FeedItem.php +++ /dev/null @@ -1,168 +1,1 @@ - - * @link http://www.ajaxray.com/projects/rss - */ - class FeedItem - { - private $elements = array(); //Collection of feed elements - private $version; - - /** - * Constructor - * - * @param contant (RSS1/RSS2/ATOM) RSS2 is default. - */ - function __construct($version = RSS2) - { - $this->version = $version; - } - - /** - * Add an element to elements array - * - * @access public - * @param srting The tag name of an element - * @param srting The content of tag - * @param array Attributes(if any) in 'attrName' => 'attrValue' format - * @return void - */ - public function addElement($elementName, $content, $attributes = null) - { - $this->elements[$elementName]['name'] = $elementName; - $this->elements[$elementName]['content'] = $content; - $this->elements[$elementName]['attributes'] = $attributes; - } - - /** - * Set multiple feed elements from an array. - * Elements which have attributes cannot be added by this method - * - * @access public - * @param array array of elements in 'tagName' => 'tagContent' format. - * @return void - */ - public function addElementArray($elementArray) - { - if(! is_array($elementArray)) return; - foreach ($elementArray as $elementName => $content) - { - $this->addElement($elementName, $content); - } - } - - /** - * Return the collection of elements in this feed item - * - * @access public - * @return array - */ - public function getElements() - { - return $this->elements; - } - - // Wrapper functions ------------------------------------------------------ - - /** - * Set the 'dscription' element of feed item - * - * @access public - * @param string The content of 'description' element - * @return void - */ - public function setDescription($description) - { - $tag = ($this->version == ATOM)? 'summary' : 'description'; - $this->addElement($tag, $description); - } - - /** - * @desc Set the 'title' element of feed item - * @access public - * @param string The content of 'title' element - * @return void - */ - public function setTitle($title) - { - $this->addElement('title', $title); - } - - /** - * Set the 'date' element of feed item - * - * @access public - * @param string The content of 'date' element - * @return void - */ - public function setDate($date) - { - if(! is_numeric($date)) - { - $date = strtotime($date); - } - - if($this->version == ATOM) - { - $tag = 'updated'; - $value = date(DATE_ATOM, $date); - } - elseif($this->version == RSS2) - { - $tag = 'pubDate'; - $value = date(DATE_RSS, $date); - } - else - { - $tag = 'dc:date'; - $value = date("Y-m-d", $date); - } - - $this->addElement($tag, $value); - } - - /** - * Set the 'link' element of feed item - * - * @access public - * @param string The content of 'link' element - * @return void - */ - public function setLink($link) - { - if($this->version == RSS2 || $this->version == RSS1) - { - $this->addElement('link', $link); - } - else - { - $this->addElement('link','',array('href'=>$link)); - $this->addElement('id', FeedWriter::uuid($link,'urn:uuid:')); - } - - } - - /** - * Set the 'encloser' element of feed item - * For RSS 2.0 only - * - * @access public - * @param string The url attribute of encloser tag - * @param string The length attribute of encloser tag - * @param string The type attribute of encloser tag - * @return void - */ - public function setEncloser($url, $length, $type) - { - $attributes = array('url'=>$url, 'length'=>$length, 'type'=>$type); - $this->addElement('enclosure','',$attributes); - } - - } // end of class FeedItem -?> --- a/lib/FeedWriter.php +++ /dev/null @@ -1,435 +1,1 @@ - - * @link http://www.ajaxray.com/projects/rss - */ - class FeedWriter - { - private $channels = array(); // Collection of channel elements - private $items = array(); // Collection of items as object of FeedItem class. - private $data = array(); // Store some other version wise data - private $CDATAEncoding = array(); // The tag names which have to encoded as CDATA - - private $version = null; - - /** - * Constructor - * - * @param constant the version constant (RSS1/RSS2/ATOM). - */ - function __construct($version = RSS2) - { - $this->version = $version; - - // Setting default value for assential channel elements - $this->channels['title'] = $version . ' Feed'; - $this->channels['link'] = 'http://www.ajaxray.com/blog'; - - //Tag names to encode in CDATA - $this->CDATAEncoding = array('description', 'content:encoded', 'summary'); - } - - // Start # public functions --------------------------------------------- - - /** - * Set a channel element - * @access public - * @param srting name of the channel tag - * @param string content of the channel tag - * @return void - */ - public function setChannelElement($elementName, $content) - { - $this->channels[$elementName] = $content ; - } - - /** - * Set multiple channel elements from an array. Array elements - * should be 'channelName' => 'channelContent' format. - * - * @access public - * @param array array of channels - * @return void - */ - public function setChannelElementsFromArray($elementArray) - { - if(! is_array($elementArray)) return; - foreach ($elementArray as $elementName => $content) - { - $this->setChannelElement($elementName, $content); - } - } - - /** - * Genarate the actual RSS/ATOM file - * - * @access public - * @return void - */ - public function genarateFeed() - { - header("Content-type: text/xml"); - - $this->printHead(); - $this->printChannels(); - $this->printItems(); - $this->printTale(); - } - - /** - * Create a new FeedItem. - * - * @access public - * @return object instance of FeedItem class - */ - public function createNewItem() - { - $Item = new FeedItem($this->version); - return $Item; - } - - /** - * Add a FeedItem to the main class - * - * @access public - * @param object instance of FeedItem class - * @return void - */ - public function addItem($feedItem) - { - $this->items[] = $feedItem; - } - - - // Wrapper functions ------------------------------------------------------------------- - - /** - * Set the 'title' channel element - * - * @access public - * @param srting value of 'title' channel tag - * @return void - */ - public function setTitle($title) - { - $this->setChannelElement('title', $title); - } - - /** - * Set the 'description' channel element - * - * @access public - * @param srting value of 'description' channel tag - * @return void - */ - public function setDescription($desciption) - { - $this->setChannelElement('description', $desciption); - } - - /** - * Set the 'link' channel element - * - * @access public - * @param srting value of 'link' channel tag - * @return void - */ - public function setLink($link) - { - $this->setChannelElement('link', $link); - } - - /** - * Set the 'image' channel element - * - * @access public - * @param srting title of image - * @param srting link url of the imahe - * @param srting path url of the image - * @return void - */ - public function setImage($title, $link, $url) - { - $this->setChannelElement('image', array('title'=>$title, 'link'=>$link, 'url'=>$url)); - } - - /** - * Set the 'about' channel element. Only for RSS 1.0 - * - * @access public - * @param srting value of 'about' channel tag - * @return void - */ - public function setChannelAbout($url) - { - $this->data['ChannelAbout'] = $url; - } - - /** - * Genarates an UUID - * @author Anis uddin Ahmad - * @param string an optional prefix - * @return string the formated uuid - */ - public function uuid($key = null, $prefix = '') - { - $key = ($key == null)? uniqid(rand()) : $key; - $chars = md5($key); - $uuid = substr($chars,0,8) . '-'; - $uuid .= substr($chars,8,4) . '-'; - $uuid .= substr($chars,12,4) . '-'; - $uuid .= substr($chars,16,4) . '-'; - $uuid .= substr($chars,20,12); - - return $prefix . $uuid; - } - // End # public functions ---------------------------------------------- - - // Start # private functions ---------------------------------------------- - - /** - * Prints the xml and rss namespace - * - * @access private - * @return void - */ - private function printHead() - { - $out = '' . "\n"; - - if($this->version == RSS2) - { - $out .= '' . PHP_EOL; - } - elseif($this->version == RSS1) - { - $out .= '' . PHP_EOL;; - } - else if($this->version == ATOM) - { - $out .= '' . PHP_EOL;; - } - echo $out; - } - - /** - * Closes the open tags at the end of file - * - * @access private - * @return void - */ - private function printTale() - { - if($this->version == RSS2) - { - echo '' . PHP_EOL . ''; - } - elseif($this->version == RSS1) - { - echo ''; - } - else if($this->version == ATOM) - { - echo ''; - } - - } - - /** - * Creates a single node as xml format - * - * @access private - * @param srting name of the tag - * @param mixed tag value as string or array of nested tags in 'tagName' => 'tagValue' format - * @param array Attributes(if any) in 'attrName' => 'attrValue' format - * @return string formatted xml tag - */ - private function makeNode($tagName, $tagContent, $attributes = null) - { - $nodeText = ''; - $attrText = ''; - - if(is_array($attributes)) - { - foreach ($attributes as $key => $value) - { - $attrText .= " $key=\"$value\" "; - } - } - - if(is_array($tagContent) && $this->version == RSS1) - { - $attrText = ' rdf:parseType="Resource"'; - } - - - $attrText .= (in_array($tagName, $this->CDATAEncoding) && $this->version == ATOM)? ' type="html" ' : ''; - $nodeText .= (in_array($tagName, $this->CDATAEncoding))? "<{$tagName}{$attrText}>"; - - if(is_array($tagContent)) - { - foreach ($tagContent as $key => $value) - { - $nodeText .= $this->makeNode($key, $value); - } - } - else - { - $nodeText .= (in_array($tagName, $this->CDATAEncoding))? $tagContent : htmlentities($tagContent); - } - - $nodeText .= (in_array($tagName, $this->CDATAEncoding))? "]]>" : ""; - - return $nodeText . PHP_EOL; - } - - /** - * @desc Print channels - * @access private - * @return void - */ - private function printChannels() - { - //Start channel tag - switch ($this->version) - { - case RSS2: - echo '' . PHP_EOL; - break; - case RSS1: - echo (isset($this->data['ChannelAbout']))? "data['ChannelAbout']}\">" : "channels['link']}\">"; - break; - } - - //Print Items of channel - foreach ($this->channels as $key => $value) - { - if($this->version == ATOM && $key == 'link') - { - // ATOM prints link element as href attribute - echo $this->makeNode($key,'',array('href'=>$value)); - //Add the id for ATOM - echo $this->makeNode('id',$this->uuid($value,'urn:uuid:')); - } - else - { - echo $this->makeNode($key, $value); - } - - } - - //RSS 1.0 have special tag with channel - if($this->version == RSS1) - { - echo "" . PHP_EOL . "" . PHP_EOL; - foreach ($this->items as $item) - { - $thisItems = $item->getElements(); - echo "" . PHP_EOL; - } - echo "" . PHP_EOL . "" . PHP_EOL . "" . PHP_EOL; - } - } - - /** - * Prints formatted feed items - * - * @access private - * @return void - */ - private function printItems() - { - foreach ($this->items as $item) - { - $thisItems = $item->getElements(); - - //the argument is printed as rdf:about attribute of item in rss 1.0 - echo $this->startItem($thisItems['link']['content']); - - foreach ($thisItems as $feedItem ) - { - echo $this->makeNode($feedItem['name'], $feedItem['content'], $feedItem['attributes']); - } - echo $this->endItem(); - } - } - - /** - * Make the starting tag of channels - * - * @access private - * @param srting The vale of about tag which is used for only RSS 1.0 - * @return void - */ - private function startItem($about = false) - { - if($this->version == RSS2) - { - echo '' . PHP_EOL; - } - elseif($this->version == RSS1) - { - if($about) - { - echo "" . PHP_EOL; - } - else - { - die('link element is not set .\n It\'s required for RSS 1.0 to be used as about attribute of item'); - } - } - else if($this->version == ATOM) - { - echo "" . PHP_EOL; - } - } - - /** - * Closes feed item tag - * - * @access private - * @return void - */ - private function endItem() - { - if($this->version == RSS2 || $this->version == RSS1) - { - echo '' . PHP_EOL; - } - else if($this->version == ATOM) - { - echo "" . PHP_EOL; - } - } - - - - // End # private functions ---------------------------------------------- - - } // end of class FeedWriter - -// autoload classes -function __autoload($class_name) -{ - require_once $class_name . '.php'; -} +