From: Maxious Date: Tue, 16 Jul 2013 10:29:44 +0000 Subject: Merge branch 'master' of ssh://apples.lambdacomplex.org/git/disclosr X-Git-Url: http://maxious.lambdacomplex.org/git/?p=disclosr.git&a=commitdiff&h=f5007cb041c9b48f6948dc6591180e1fcc84ce51 --- Merge branch 'master' of ssh://apples.lambdacomplex.org/git/disclosr Conflicts: documents/sitemap.xml.php Former-commit-id: 0480844aebe20a3af30b1c98aa192a5a823fb4d6 --- --- a/admin/logo.svg +++ b/admin/logo.svg @@ -1,34 +1,34 @@ - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + --- a/admin/refreshDesignDoc.php +++ b/admin/refreshDesignDoc.php @@ -8,21 +8,13 @@ $obj->_id = "_design/" . urlencode("app"); $obj->language = "javascript"; $obj->views->all->map = "function(doc) { emit(doc._id, doc); };"; -$obj->views->byDate->map = "function(doc) { if (doc.title != "Disclosure Log Updated") { emit(doc.date, doc); } };"; +$obj->views->byDate->map = "function(doc) { if (doc.title != \"Disclosure Log Updated\") { emit(doc.date, doc); } };"; $obj->views->byDateMonthYear->map = "function(doc) { emit(doc.date, doc); };"; $obj->views->byDateMonthYear->reduce = "_count"; $obj->views->byAgencyID->map = "function(doc) { emit(doc.agencyID, doc); };"; $obj->views->byAgencyID->reduce = "_count"; -$obj->views->fieldNames->map = ' -function(doc) { -for(var propName in doc) { - emit(propName, doc._id); - } - -}'; -$obj->views->fieldNames->reduce = 'function (key, values, rereduce) { - return values.length; -}'; +$obj->views->fieldNames->map = 'function(doc) { for(var propName in doc) { emit(propName, doc._id); }}'; +$obj->views->fieldNames->reduce = 'function (key, values, rereduce) { return values.length; }'; // allow safe updates (even if slightly slower due to extra: rev-detection check). $foidb->save($obj, true); --- a/documents/robots.txt +++ b/documents/robots.txt @@ -3,4 +3,5 @@ User-agent: * Disallow: /admin/ +Disallow: /viewDocument.php Sitemap: http://disclosurelo.gs/sitemap.xml.php --- a/documents/scrapers/f5ce2d1651739704634eb8ca4b2b46d3.py +++ b/documents/scrapers/f5ce2d1651739704634eb8ca4b2b46d3.py @@ -7,12 +7,12 @@ #http://www.doughellmann.com/PyMOTW/abc/ class ScraperImplementation(genericScrapers.GenericOAICDisclogScraper): def getTable(self,soup): - return soup.find(id = "ctl00_PlaceHolderMain_PublishingPageContent__ControlWrapper_RichHtmlField").table + return soup.find(id = "block-system-main").table def getColumnCount(self): - return 7 + return 2 def getColumns(self,columns): - (id, date, title, description,link,deldate, notes) = columns - return (id, date, title, description, notes) + (date, title) = columns + return (date, date, title, title, None) if __name__ == '__main__': print 'Subclass:', issubclass(ScraperImplementation, genericScrapers.GenericOAICDisclogScraper) --- a/documents/sitemap.xml.php +++ b/documents/sitemap.xml.php @@ -1,30 +1,48 @@ "; echo '' . "\n"; echo " " . local_url() . "index.php1.0\n"; foreach (scandir("./") as $file) { - if (strpos($file, ".php") !== false && $file != "index.php" && $file != "sitemap.xml.php") + if (strpos($file, ".php") !== false && $file != "index.php" && $file != "sitemap.xml.php"&& $file != "viewDocument.php") echo " " . local_url() . "$file0.6\n"; + } } $agenciesdb = $server->get_db('disclosr-agencies'); +$foidocsdb = $server->get_db('disclosr-foidocuments'); try { $rows = $agenciesdb->get_view("app", "byCanonicalName")->rows; foreach ($rows as $row) { echo '' . local_url() . 'agency.php?id=' . $row->value->_id . "0.3\n"; } + unset($rows); + $rows = null; } catch (SetteeRestClientException $e) { setteErrorHandler($e); } -$foidocsdb = $server->get_db('disclosr-foidocuments'); + +foreach (range(0, 8) as $number) { try { - $rows = $foidocsdb->get_view("app", "all")->rows; + $rows = $foidocsdb->get_view("app", "all", Array($number,$number+1))->rows; foreach ($rows as $row) { echo '' . local_url() . 'view.php?id=' . $row->value->_id . "0.3\n"; } + unset($rows); + $rows = null; +} catch (SetteeRestClientException $e) { + setteErrorHandler($e); +} +} + +try { + $rows = $foidocsdb->get_view("app", "all", Array('9','fffffffff'))->rows; + foreach ($rows as $row) { + echo '' . local_url() . 'view.php?id=' . $row->value->_id . "0.3\n"; + } + unset($rows); + $rows = null; } catch (SetteeRestClientException $e) { setteErrorHandler($e); } --- /dev/null +++ b/exportAgencies.csv.php @@ -1,1 +1,88 @@ +get_db('disclosr-agencies'); + + +$foiEmail = Array(); +try { + $rows = $db->get_view("app", "foiEmails", null, true)->rows; + //print_r($rows); + foreach ($rows as $row) { + $foiEmail[$row->key] = $row->value; + } +} catch (SetteeRestClientException $e) { + setteErrorHandler($e); + die(); +} + +$fp = fopen('php://output', 'w'); +if ($fp && $db) { + header('Content-Type: text/csv; charset=utf-8'); + header('Content-Disposition: attachment; filename="export.' . date("c") . '.csv"'); + header('Pragma: no-cache'); + header('Expires: 0'); + fputcsv($fp, $headers); + try { + $agencies = $db->get_view("app", "byCanonicalName", null, true)->rows; + //print_r($rows); + foreach ($agencies as $agency) { + // print_r($agency); + + if (isset($agency->value->foiEmail) && $agency->value->foiEmail != "null" && !isset($agency->value->status)) { + $row = Array(); + $row["title"] = trim($agency->value->name); + $row["abbr"] = (isset($agency->value->shortName) ? $agency->value->shortName : ""); + $row["key"] = (isset($agency->value->shortName) ? "au/".strtolower($agency->value->shortName) : ""); + $row["category"] =""; + $row["parent"] =""; + $row["parentkey"] =""; + $row["description"] = (isset($agency->value->description) ? $agency->value->description : ""); + $row["url"] = (isset($agency->value->website) ? $agency->value->website : ""); + $row["jurisdiction"] = "Australia"; + $row["jurisdiction_code"] = "au"; + + $row["source"] =""; + $row["source_url"] =""; + $row["address"] =""; + $row["contact"] =""; + + $row["email"] = (isset($agency->value->foiEmail) ? $agency->value->foiEmail : ""); + $row["tags"] =""; + $row["created_at"] =""; + $row["updated_at"] =""; + + + $otherBodies = Array(); + if (isset($agency->value->foiBodies)) { + $otherBodies = array_merge($otherBodies, $agency->value->foiBodies); + } + if (isset($agency->value->positions)) { + $positions = Array(); + foreach ($agency->value->positions as $position) { + $positions[] = "Office of the ".$position; + } + $otherBodies = array_merge($otherBodies, $positions); + } + sort($otherBodies); + if (count($otherBodies) > 0) { + $row["description"] .= "
This department also responds to requests for information held by " . implode(", ", $otherBodies); + } + + + fputcsv($fp, array_values($row)); + } + } + } catch (SetteeRestClientException $e) { + setteErrorHandler($e); + } + + die; +} +?> + --- a/getAgency.php +++ b/getAgency.php @@ -188,7 +188,7 @@
'; - echo '
'; + echo '
'; echo '

' . $row['name'] . '

'; if (isset($row['description'])) { --- a/include/common.inc.php +++ b/include/common.inc.php @@ -10,6 +10,7 @@ || strstr($_SERVER['PHP_SELF'], "documents/") || $_SERVER['SERVER_NAME'] == "disclosurelo.gs" || $_SERVER['SERVER_NAME'] == "www.disclosurelo.gs" + || $_SERVER['SERVER_NAME'] == "direct.disclosurelo.gs" ) $basePath = "../"; --- a/include/template.inc.php +++ b/include/template.inc.php @@ -44,7 +44,7 @@ - +