From: Maxious Date: Fri, 10 Feb 2012 02:01:57 +0000 Subject: All data export in CSV X-Git-Url: https://maxious.lambdacomplex.org/git/?p=disclosr.git&a=commitdiff&h=c2b44bf9e939dd331b609568df5a5f2db66859a9 --- All data export in CSV Former-commit-id: 0541eb6c63de81151dbeef836028d0bbb32a680e --- --- /dev/null +++ b/admin/exportAll.csv.php @@ -1,1 +1,63 @@ +get_db('disclosr-agencies'); +$headers = Array(); + try { + $rows = $db->get_view("app", "fieldNames?group=true", null, true)->rows; + + $dataValues = Array(); + foreach ($rows as $row) { + $headers[] = $row->key; + } +} catch (SetteeRestClientException $e) { + setteErrorHandler($e); +} + +$fp = fopen('php://output', 'w'); +if ($fp && $db) { + header('Content-Type: text/csv; charset=utf-8'); + header('Content-Disposition: attachment; filename="export.' . date("c") . '.csv"'); + header('Pragma: no-cache'); + header('Expires: 0'); + fputcsv($fp, $headers); + try { + $agencies = $db->get_view("app", "byCanonicalName", null, true)->rows; + //print_r($rows); + foreach ($agencies as $agency) { + // print_r($agency); + + if ( !isset($agency->value->status)) { + $row = Array(); + $agencyArray = object_to_array($agency->value); + foreach ($headers as $fieldName) { + if (isset($agencyArray[$fieldName])) { + if (is_array($agencyArray[$fieldName])) { + $row[] = implode(";",$agencyArray[$fieldName]); + } else { + $row[] = $agencyArray[$fieldName]; + } + } else { + $row[] = ""; + } + } + + fputcsv($fp, array_values($row)); + + + } + } + } catch (SetteeRestClientException $e) { + setteErrorHandler($e); + } + + die; +} +?> + --- /dev/null +++ b/couchdb/SetteeDatabase.class.php @@ -1,1 +1,306 @@ - +conn_url = $conn_url; + $this->dbname = $dbname; + $this->rest_client = SetteeRestClient::get_instance($this->conn_url); + } + + + /** + * Get UUID from CouchDB + * + * @return + * CouchDB-generated UUID string + * + */ + function gen_uuid() { + $ret = $this->rest_client->http_get('_uuids'); + return $ret['decoded']->uuids[0]; // should never be empty at this point, so no checking + } + + /** + * Create or update a document database + * + * @param $document + * PHP object, a PHP associative array, or a JSON String representing the document to be saved. PHP Objects and arrays are JSON-encoded automatically. + * + *

If $document has a an "_id" property set, it will be used as document's unique id (even for "create" operation). + * If "_id" is missing, CouchDB will be used to generate a UUID. + * + *

If $document has a "_rev" property (revision), document will be updated, rather than creating a new document. + * You have to provide "_rev" if you want to update an existing document, otherwise operation will be assumed to be + * one of creation and you will get a duplicate document exception from CouchDB. Also, you may not provide "_rev" but + * not provide "_id" since that is an invalid input. + * + * @param $allowRevAutoDetection + * Default: false. When true and _rev is missing from the document, save() function will auto-detect latest revision + * for a document and use it. This option is "false" by default because it involves an extra http HEAD request and + * therefore can make save() operation slightly slower if such auto-detection is not required. + * + * @return + * document object with the database id (uuid) and revision attached; + * + * @throws SetteeCreateDatabaseException + */ + function save($document, $allowRevAutoDetection = false) { + if (is_string($document)) { + $document = json_decode($document); + } + + // Allow passing of $document as an array (for syntactic simplicity and also because in JSON world it does not matter) + if(is_array($document)) { + $document = (object) $document; + } + + if (empty($document->_id) && empty($document->_rev)) { + $id = $this->gen_uuid(); + } + elseif (empty($document->_id) && !empty($document->_rev)) { + throw new SetteeWrongInputException("Error: You can not save a document with a revision provided, but missing id"); + } + else { + $id = $document->_id; + + if ($allowRevAutoDetection) { + try { + $rev = $this->get_rev($id); + } catch (SetteeRestClientException $e) { + // auto-detection may fail legitimately, if a document has never been saved before (new doc), so skipping error + } + if (!empty($rev)) { + $document->_rev = $rev; + } + } + } + + $full_uri = $this->dbname . "/" . $this->safe_urlencode($id); + $document_json = json_encode($document, JSON_NUMERIC_CHECK); + + $ret = $this->rest_client->http_put($full_uri, $document_json); + + $document->_id = $ret['decoded']->id; + $document->_rev = $ret['decoded']->rev; + + return $document; + } + + /** + * @param $doc + * @param $name + * @param $content + * Content of the attachment in a string-buffer format. This function will automatically base64-encode content for + * you, so you don't have to do it. + * @param $mime_type + * Optional. Will be auto-detected if not provided + * @return void + */ + public function add_attachment($doc, $name, $content, $mime_type = null) { + if (empty($doc->_attachments) || !is_object($doc->_attachments)) { + $doc->_attachments = new stdClass(); + } + + if (empty($mime_type)) { + $mime_type = $this->rest_client->content_mime_type($content); + } + + $doc->_attachments->$name = new stdClass(); + $doc->_attachments->$name->content_type = $mime_type; + $doc->_attachments->$name->data = base64_encode($content); + } + + /** + * @param $doc + * @param $name + * @param $file + * Full path to a file (e.g. as returned by PHP's realpath function). + * @param $mime_type + * Optional. Will be auto-detected if not provided + * @return void + */ + public function add_attachment_file($doc, $name, $file, $mime_type = null) { + $content = file_get_contents($file); + $this->add_attachment($doc, $name, $content, $mime_type); + } + + /** + * + * Retrieve a document from CouchDB + * + * @throws SetteeWrongInputException + * + * @param $id + * Unique ID (usually: UUID) of the document to be retrieved. + * @return + * database document in PHP object format. + */ + function get($id) { + if (empty($id)) { + throw new SetteeWrongInputException("Error: Can't retrieve a document without a uuid."); + } + + $full_uri = $this->dbname . "/" . $this->safe_urlencode($id); +$full_uri = str_replace("%3Frev%3D","?rev=",$full_uri); + $ret = $this->rest_client->http_get($full_uri); + return $ret['decoded']; + } + + /** + * + * Get the latest revision of a document with document id: $id in CouchDB. + * + * @throws SetteeWrongInputException + * + * @param $id + * Unique ID (usually: UUID) of the document to be retrieved. + * @return + * database document in PHP object format. + */ + function get_rev($id) { + if (empty($id)) { + throw new SetteeWrongInputException("Error: Can't query a document without a uuid."); + } + + $full_uri = $this->dbname . "/" . $this->safe_urlencode($id); + $headers = $this->rest_client->http_head($full_uri); + if (empty($headers['Etag'])) { + throw new SetteeRestClientException("Error: could not retrieve revision. Server unexpectedly returned empty Etag"); + } + $etag = str_replace('"', '', $headers['Etag']); + return $etag; + } + + /** + * Delete a document + * + * @param $document + * a PHP object or JSON representation of the document that has _id and _rev fields. + * + * @return void + */ + function delete($document) { + if (!is_object($document)) { + $document = json_decode($document); + } + + $full_uri = $this->dbname . "/" . $this->safe_urlencode($document->_id) . "?rev=" . $document->_rev; + $this->rest_client->http_delete($full_uri); + } + + + /*----------------- View-related functions --------------*/ + + /** + * Create a new view or update an existing one. + * + * @param $design_doc + * @param $view_name + * @param $map_src + * Source code of the map function in Javascript + * @param $reduce_src + * Source code of the reduce function in Javascript (optional) + * @return void + */ + function save_view($design_doc, $view_name, $map_src, $reduce_src = null) { + $obj = new stdClass(); + $obj->_id = "_design/" . urlencode($design_doc); + $view_name = urlencode($view_name); + $obj->views->$view_name->map = $map_src; + if (!empty($reduce_src)) { + $obj->views->$view_name->reduce = $reduce_src; + } + + // allow safe updates (even if slightly slower due to extra: rev-detection check). + return $this->save($obj, true); + } + + /** + * Create a new view or update an existing one. + * + * @param $design_doc + * @param $view_name + * @param $key + * key parameter to a view. Can be a single value or an array (for a range). If passed an array, function assumes + * that first element is startkey, second: endkey. + * @param $descending + * return results in descending order. Please don't forget that if you are using a startkey/endkey, when you change + * order you also need to swap startkey and endkey values! + * + * @return void + */ + function get_view($design_doc, $view_name, $key = null, $descending = false) { + $id = "_design/" . urlencode($design_doc); + $view_name = urlencode($view_name); + $id .= "/_view/$view_name"; + + $data = array(); + if (!empty($key)) { + if (is_string($key)) { + $data = "key=" . '"' . $key . '"'; + } + elseif (is_array($key)) { + list($startkey, $endkey) = $key; + $data = "startkey=" . '"' . $startkey . '"&' . "endkey=" . '"' . $endkey . '"'; + } + + if ($descending) { + $data .= "&descending=true"; + } + } + + + + if (empty($id)) { + throw new SetteeWrongInputException("Error: Can't retrieve a document without a uuid."); + } + + $full_uri = $this->dbname . "/" . $this->safe_urlencode($id); +$full_uri = str_replace("%253Fgroup%253Dtrue","?group=true",$full_uri); + $ret = $this->rest_client->http_get($full_uri, $data); + return $ret['decoded']; + + } + + /** + * @param $id + * @return + * return a properly url-encoded id. + */ + private function safe_urlencode($id) { + //-- System views like _design can have "/" in their URLs. + $id = rawurlencode($id); + if (substr($id, 0, 1) == '_') { + $id = str_replace('%2F', '/', $id); + } + return $id; + } + + /** Getter for a database name */ + function get_name() { + return $this->dbname; + } + +} --- a/getAgency.php +++ b/getAgency.php @@ -58,9 +58,7 @@ echo ""; } echo" "; - } else if (strpos($key, "has") === 0) { - echo ""; - } else { + } else { echo ""; if ((strpos($key, "URL") > 0 || $key == 'website') && $value != "") { echo "view"; @@ -80,11 +78,9 @@ foreach ($defaultFields as $defaultField) { if (!isset($row[$defaultField])) { if ($schemas['agency']['properties'][$defaultField]['type'] == "string") { - if (strpos($defaultField, "has") === 0) { - $row[$defaultField] = "false"; - } else { + $row[$defaultField] = ""; - } + } if ($schemas['agency']['properties'][$defaultField]['type'] == "array") { --- a/include/couchdb.inc.php +++ b/include/couchdb.inc.php @@ -82,24 +82,39 @@ emit("total", 1); } }'; - $obj->views->score->map = 'if(!String.prototype.startsWith){ + $obj->views->scoreHas->map = 'if(!String.prototype.startsWith){ String.prototype.startsWith = function (str) { return !this.indexOf(str); } } - +if(!String.prototype.endsWith){ + String.prototype.endsWith = function(suffix) { +     return this.indexOf(suffix, this.length - suffix.length) !== -1; + }; +} function(doc) { -count = 0; if (typeof(doc["status"]) == "undefined" || doc["status"] != "suspended") { for(var propName in doc) { - if(typeof(doc[propName]) != "undefined" && propName.startsWith("l")) { - count++ + if(typeof(doc[propName]) != "undefined" && (propName.startsWith("has") || propName.endsWith("URL"))) { + emit(propName, 1); } } - emit(count+doc._id, {id:doc._id, name: doc.name, score:count}); + emit("total", 1); } }'; - + $obj->views->scoreHas->reduce = 'function (key, values, rereduce) { + return sum(values); +}'; + $obj->views->fieldNames->map = ' +function(doc) { +for(var propName in doc) { + emit(propName, doc._id); + } + +}'; + $obj->views->fieldNames->reduce = 'function (key, values, rereduce) { + return values.length; +}'; // allow safe updates (even if slightly slower due to extra: rev-detection check). return $db->save($obj, true); } --- a/schemas/agency.json.php +++ b/schemas/agency.json.php @@ -26,17 +26,28 @@ "recordsListURL" => Array("type" => "string", "required" => true, "x-title" => "Files/Records Held", "description" => "Indexed lists of departmental and agency files, mandated by the Senate"), "FOIDocumentsURL" => Array("type" => "string", "required" => true, "x-title" => "FOI Documents Released", "description" => "FOI Disclosure Log URL"), "FOIDocumentsRSSURL" => Array("type" => "string", "required" => false, "x-title" => "RSS Feed of FOI Documents Released", "description" => "FOI Disclosure Log in RSS format"), - "hasFOIPDF" => Array("type" => "string", "required" => false, "x-title" => "Has FOI Documents Released in PDF", "description" => "FOI Disclosure Log contains any PDFs"), + "hasFOIPDF" => Array("type" => "array", "required" => false, "x-title" => "Has FOI Documents Released in PDF", "description" => "FOI Disclosure Log contains any PDFs", + "items" => Array("type" => "string")), "infoPublicationSchemeURL" => Array("type" => "string", "required" => true, "x-title" => "Information Publication Scheme", "description" => ""), "appointmentsURL" => Array("type" => "string", "required" => true, "x-title" => "Agency Appointments/Boards", "description" => "Departmental and agency appointments and vacancies , mandated by the Senate"), "advertisingURL" => Array("type" => "string", "required" => true, "x-title" => "Approved Advertising Campaigns", "description" => " Agency advertising and public information projects, mandated by the Senate "), - "hasRSS" => Array("type" => "string", "required" => true, "x-title" => "Has RSS", "description" => ""), - "hasMailingList" => Array("type" => "string", "required" => true, "x-title" => "Has Mailing List", "description" => ""), - "hasTwitter" => Array("type" => "string", "required" => true, "x-title" => "Has Twitter", "description" => ""), - "hasFacebook" => Array("type" => "string", "required" => true, "x-title" => "Has Facebook", "description" => ""), - "hasYouTube" => Array("type" => "string", "required" => true, "x-title" => "Has YouTube", "description" => ""), - "hasFlickr" => Array("type" => "string", "required" => true, "x-title" => "Has Flickr", "description" => ""), - "hasCCBY" => Array("type" => "string", "required" => true, "x-title" => "Has CC-BY", "description" => "Has any page licenced Creative Commons - Attribution"), + "hasRSS" => Array("type" => "array", "required" => true, "x-title" => "Has RSS", "description" => ""), + "hasMailingList" => Array("type" => "array", "required" => true, "x-title" => "Has Mailing List", "description" => "", + "items" => Array("type" => "string")), + "hasTwitter" => Array("type" => "array", "required" => true, "x-title" => "Has Twitter", "description" => "", + "items" => Array("type" => "string")), + "hasFacebook" => Array("type" => "array", "required" => true, "x-title" => "Has Facebook", "description" => "", + "items" => Array("type" => "string")), + "hasYouTube" => Array("type" => "array", "required" => true, "x-title" => "Has YouTube", "description" => "", + "items" => Array("type" => "string")), + "hasFlickr" => Array("type" => "array", "required" => true, "x-title" => "Has Flickr", "description" => "", + "items" => Array("type" => "string")), + "hasCCBY" => Array("type" => "array", "required" => true, "x-title" => "Has CC-BY", "description" => "Has any page licenced Creative Commons - Attribution", + "items" => Array("type" => "string")), + "hasRestrictiveLicence" => Array("type" => "array","required" => true, "x-title" => "Has Restrictive Licence", "description" => "Has any page licenced under terms more restrictive than Crown Copyright", + "items" => Array("type" => "string")), + "hasCrownCopyright" => Array("type" => "array", "required" => true, "x-title" => "Has Standard Crown Copyright licence", "description" => "Has any page still licenced under the former Commonwealth Copyright Administration", + "items" => Array("type" => "string")), ), /* "org":{"type":"object", "properties":{ --- a/scrape.py +++ b/scrape.py @@ -15,13 +15,25 @@ addinfourl.code = code return addinfourl -def fetchURL(docsdb, url, agencyID): +def fetchURL(docsdb, url, fieldName, agencyID, scrape_again=True): hash = hashlib.md5(url).hexdigest() req = urllib2.Request(url) - print "Fetching %s", url + print "Fetching %s" % url doc = docsdb.get(hash) if doc == None: - doc = {'_id': hash, 'agencyID': agencyID} + doc = {'_id': hash, 'agencyID': agencyID, 'url': url, 'fieldName':fieldName} + else: + if (time.time() - doc['page_scraped']) < 3600: + print "Uh oh, trying to scrape URL again too soon!" + last_attachment_fname = doc["_attachments"].keys()[-1] + last_attachment = docsdb.get_attachment(doc,last_attachment_fname) + return (doc['mime_type'],last_attachment) + if scrape_again == False: + print "Not scraping this URL again as requested" + return (None,None) + + time.sleep(3) # wait 3 seconds to give webserver time to recover + #if there is a previous version stored in couchdb, load caching helper tags if doc.has_key('etag'): req.add_header("If-None-Match", doc['etag']) @@ -33,6 +45,8 @@ headers = url_handle.info() # the addinfourls have the .info() too doc['etag'] = headers.getheader("ETag") doc['last_modified'] = headers.getheader("Last-Modified") + doc['date'] = headers.getheader("Date") + doc['page_scraped'] = time.time() doc['web_server'] = headers.getheader("Server") doc['powered_by'] = headers.getheader("X-Powered-By") doc['file_size'] = headers.getheader("Content-Length") @@ -40,7 +54,7 @@ if hasattr(url_handle, 'code'): if url_handle.code == 304: print "the web page has not been modified" - return None + return (None,None) else: content = url_handle.read() docsdb.save(doc) @@ -49,15 +63,16 @@ return (doc['mime_type'], content) #store as attachment epoch-filename else: - print "error %s in downloading %s", url_handle.code, URL - #record/alert error to error database - + print "error %s in downloading %s" % url_handle.code, URL + doc['error'] = "error %s in downloading %s" % url_handle.code, URL + docsdb.save(doc) + return (None,None) -def scrapeAndStore(docsdb, url, depth, agencyID): - (mime_type,content) = fetchURL(docsdb, url, agencyID) - if content != None: +def scrapeAndStore(docsdb, url, depth, fieldName, agencyID): + (mime_type,content) = fetchURL(docsdb, url, fieldName, agencyID) + if content != None and depth > 0: if mime_type == "text/html" or mime_type == "application/xhtml+xml" or mime_type =="application/xml": # http://www.crummy.com/software/BeautifulSoup/documentation.html soup = BeautifulSoup(content) @@ -70,21 +85,18 @@ print "Removing element", nav['class'] nav.extract() links = soup.findAll('a') # soup.findAll('a', id=re.compile("^p-")) + linkurls = set([]) for link in links: if link.has_key("href"): if link['href'].startswith("http"): - linkurl = link['href'] + # lets not do external links for now + # linkurls.add(link['href']) + None else: - linkurl = urljoin(url,link['href']) - print linkurl - #for each unique link - # if - #if html mimetype - # go down X levels, - # diff with last stored attachment, store in document - #if not - # remember to save parentURL and title (link text that lead to document) - + linkurls.add(urljoin(url,link['href'].replace(" ","%20"))) + for linkurl in linkurls: + #print linkurl + scrapeAndStore(docsdb, linkurl, depth-1, fieldName, agencyID) couch = couchdb.Server('http://127.0.0.1:5984/') @@ -95,5 +107,10 @@ for row in agencydb.view('app/getScrapeRequired'): #not recently scraped agencies view? agency = agencydb.get(row.id) print agency['name'] - scrapeAndStore(docsdb, agency['website'],1,agency['_id']) + for key in agency.keys(): + if key == 'website' or key.endswith('URL'): + print key + scrapeAndStore(docsdb, agency[key],agency['scrapeDepth'],key,agency['_id']) + agency['metadata']['lastscraped'] = time.time() + agencydb.save(agency)