Add public ranking listing, link charts better
Former-commit-id: 889d2d51a7712e7454f582b671b6142d619ad672
--- a/about.php
+++ b/about.php
@@ -5,6 +5,10 @@
<div class="foundation-header">
<h1><a href="about.php">About/FAQ</a></h1>
</div>
+<a href="http://thenounproject.com/noun/document-dump/">Document Dump icon designed by Iconathon, 2013</a>
+Contact us: maxious@lambdacomplex.org
+
+Exports: <a href="admin/exportAll.csv.php">All Agencies</a> <a href="admin/exportEmployees.csv.php">Agency Employee Headcounts</a>
<h2> Attributions </h2>
National Archives of Australia, Australian Governments’ Interactive Functions Thesaurus, 2nd edition, September 2005, published at http://www.naa.gov.au/recordkeeping/thesaurus/index.htm <br/>
data.gov.au http://data.gov.au/dataset/directory-gov-au-full-data-export/ <br/>
--- a/admin/refreshDesignDoc.php
+++ b/admin/refreshDesignDoc.php
@@ -42,6 +42,7 @@
$obj->views->byWebServer->map = "function(doc) {\n emit(doc.web_server, doc);\n}";
$obj->views->datasets->map = "function(doc) {\nif (doc.fieldName == \"data\") {\n emit(doc._id, doc);\n}\n}";
+$obj->views->datasetGroups->map = "function(doc) {\nif (doc.fieldName == \"data\") {\n doc.metadata[\"data.gov.au Category\"] && doc.metadata[\"data.gov.au Category\"].forEach(function(tag) {\n emit(tag, doc.url); \n });\n}\n}";
$obj->views->getValidationRequired->map = "function(doc) {\nif (doc.mime_type == \"text/html\" \n&& typeof(doc.validation) == \"undefined\") {\n emit(doc._id, doc._attachments);\n}\n}";
$docdb->save($obj, true);
--- a/charts.php
+++ /dev/null
@@ -1,133 +1,1 @@
-<?php
-include_once('include/common.inc.php');
-include_header('Charts');
-$db = $server->get_db('disclosr-agencies');
-?>
-<div class="foundation-header">
- <h1><a href="about.php">Charts</a></h1>
- <h4 class="subheader">Lorem ipsum.</h4>
-</div>
-<div id="scores" style="width:900px;height:500px;"></div>
-<div id="employees" style="width:1000px;height:900px;"></div>
-<script id="source">
- window.onload = function() {
- $(document).ready(function() {
- var d1 = [];
- var scorelabels = [];
- <?php
- try {
- $rows = $db->get_view("app", "scoreHas?group=true", null, true)->rows;
-
- $dataValues = Array();
- foreach ($rows as $row) {
- $dataValues[$row->value] = $row->key;
- }
- $i = 0;
- ksort($dataValues);
- foreach ($dataValues as $value => $key) {
-
- echo " d1.push([$i, $value]);" . PHP_EOL;
- echo " scorelabels.push('$key');" . PHP_EOL;
- $i++;
- }
- } catch (SetteeRestClientException $e) {
- setteErrorHandler($e);
- }
- ?>
- function scoretrackformatter(obj) {
- if (scorelabels[Math.floor(obj.x)]) {
- return (scorelabels[Math.floor(obj.x)])+"="+obj.y;
-
- } else {
- return "";
- }
- }
- function scoretickformatter(val, axis) {
- if (scorelabels[Math.floor(val)]) {
- return '<p style="margin-top:8em;-webkit-transform:rotate(-90deg);">'+(scorelabels[Math.floor(val)])+"</b>";
-
- } else {
- return "";
- }
- }
- Flotr.draw(document.getElementById("scores"), [ {data: d1}], {
- HtmlText: true,
- bars : {
- show : true
- },
- mouse : {
- track : true,
- relative : true,
- trackFormatter: scoretrackformatter
- },yaxis: {
- autoscaling: true
- },
- xaxis: {
- autoscaling: true,
- minorTickFreq: 0.6,
- noTicks : scorelabels.length,
- tickFormatter: scoretickformatter
- }
- });
-
-
-
-
-
-
-
-
-var emplabels = [];
-function emptrackformatter(obj) {
-
- return (obj.series.label)+" = "+obj.y+" in "+emplabels[Math.floor(obj.x)];
-
- }
- function emptickformatter(val, axis) {
- if (emplabels[Math.floor(val)]) {
- return '<p style="margin-top:8em;-webkit-transform:rotate(-90deg);">'+(emplabels[Math.floor(val)])+"</b>";
-
- } else {
- return "";
- }
- }
-function onDataReceived(series) {
- emplabels = series.labels;
- Flotr.draw(document.getElementById("employees"), series.data, {
- mouse : {
- track : true,
- relative : true,
- trackFormatter: emptrackformatter
- },yaxis: {
- max: 10000,
- scaling: 'logarithmic'
- },
- xaxis: {
- minorTickFreq: 1,
- noTicks: emplabels.length,
- showMinorLabels: true,
- tickFormatter: emptickformatter
- },
- legend: {
- show: false
- }
- });
- }
-
- $.ajax({
- url: "admin/exportEmployees.csv.php?format=json",
- method: 'GET',
- dataType: 'json',
- success: onDataReceived
- });
-
-
- });
- };
-
-</script>
-
-<?php
-include_footer();
-?>
--- /dev/null
+++ b/disclosr.iml
@@ -1,1 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="WEB_MODULE" version="4">
+ <component name="FacetManager">
+ <facet type="Python" name="Python">
+ <configuration sdkName="" />
+ </facet>
+ </component>
+ <component name="NewModuleRootManager" inherit-compiler-output="true">
+ <exclude-output />
+ <content url="file://$MODULE_DIR$" />
+ <orderEntry type="inheritedJdk" />
+ <orderEntry type="sourceFolder" forTests="false" />
+ </component>
+</module>
+
--- a/documents/charts.php
+++ b/documents/charts.php
@@ -13,10 +13,10 @@
?>
<div class="foundation-header">
<h1><a href="about.php">Charts</a></h1>
- <h4 class="subheader">Lorem ipsum.</h4>
+ <h4 class="subheader"></h4>
</div>
<div id="bydate" style="width:1000px;height:300px;"></div>
-<div id="byagency" style="width:1200px;height:300px;"></div>
+<div id="byagency" style="width:1200px;height:800px;"></div>
<script id="source">
window.onload = function () {
$(document).ready(function () {
@@ -59,7 +59,7 @@
mode: 'x'
},
HtmlText: false,
- title: 'Time'
+ title: 'Disclosure Log entries added by Date'
};
// Draw graph with default options, overwriting with passed options
@@ -98,12 +98,12 @@
var agencylabels = [];
function agencytrackformatter(obj) {
- return agencylabels[Math.floor(obj.x)] + " = " + obj.y;
+ return agencylabels[Math.floor(obj.y)] + " = " + obj.x;
}
function agencytickformatter(val, axis) {
if (agencylabels[Math.floor(val)]) {
- return '<p style="margin-top:8em;-webkit-transform:rotate(-90deg);">' + (agencylabels[Math.floor(val)]) + "</b>";
+ return (agencylabels[Math.floor(val)]) ;
} else {
return "";
@@ -117,7 +117,7 @@
$dataValues = Array();
$i = 0;
foreach ($rows as $row) {
- echo " d2.push([".$i.", $row->value]);" . PHP_EOL;
+ echo " d2.push([ $row->value,$i]);" . PHP_EOL;
echo " agencylabels.push(['".str_replace("'","",$idtoname[$row->key])."']);" . PHP_EOL;
$i++;
@@ -131,9 +131,10 @@
document.getElementById("byagency"),
[d2],
{
+ title: "Disclosure Log entries by Agency",
bars: {
show: true,
- horizontal: false,
+ horizontal: true,
shadowSize: 0,
barWidth: 0.5
},
@@ -143,14 +144,14 @@
trackFormatter: agencytrackformatter
},
yaxis: {
- min: 0,
- autoscaleMargin: 1
- },
- xaxis: {
minorTickFreq: 1,
noTicks: agencylabels.length,
showMinorLabels: true,
tickFormatter: agencytickformatter
+ },
+ xaxis: {
+ min: 0,
+ autoscaleMargin: 1
},
legend: {
show: false
--- /dev/null
+++ b/documents/datagov-export-groups.py
@@ -1,1 +1,81 @@
+import ckanclient
+import couchdb
+from ckanclient import CkanApiError
+import re
+
+class LoaderError(Exception):
+ pass
+
+# Instantiate the CKAN client.
+#ckan = ckanclient.CkanClient(base_location='http://localhost:5000/api', api_key='b47b24cd-591d-40c1-8677-d73101d56d1b')
+api_key = 'ff34526e-f794-4068-8235-fcbba38cd8bc'
+ckan = ckanclient.CkanClient(base_location='http://data.disclosurelo.gs/api',
+ api_key=api_key)
+couch = couchdb.Server('http://127.0.0.1:5984/')
+#couch = couchdb.Server('http://192.168.1.113:5984/')
+
+# https://github.com/okfn/ckanext-importlib
+def munge(name):
+ # convert spaces to underscores
+ name = re.sub(' ', '_', name).lower()
+ # convert symbols to dashes
+ name = re.sub('[:]', '_-', name).lower()
+ name = re.sub('[/]', '-', name).lower()
+ # take out not-allowed characters
+ name = re.sub('[^a-zA-Z0-9-_]', '', name).lower()
+ # remove double underscores
+ name = re.sub('__', '_', name).lower()
+ return name
+
+
+def name_munge(input_name):
+ return munge(input_name.replace(' ', '').replace('.', '_').replace('&', 'and'))
+
+
+docsdb = couch['disclosr-documents']
+
+if __name__ == "__main__":
+ groups = {}
+ for doc in docsdb.view('app/datasetGroups'):
+ group_name = doc.key
+ if group_name != "Not specified":
+ pkg_name = filter(lambda x: x in '0123456789abcdefghijklmnopqrstuvwxyz-_',
+ doc.value.replace("http://data.gov.au/dataset/", '').replace('/', '')[:100]);
+ if group_name in groups.keys():
+ groups[group_name] = list(set(groups[group_name] + [pkg_name]))
+ else:
+ groups[group_name] = [pkg_name]
+
+ # add dataset to group(s)
+ for group_name in groups.keys():
+ if group_name != "Not specified":
+ group_url = name_munge(group_name[:100])
+ print group_name
+ print groups[group_name]
+ try:
+ # Update the group details
+ group_entity = ckan.group_entity_get(group_url)
+ print "group "+group_name+" exists"
+ if 'packages' in group_entity.keys():
+ group_entity['packages'] = list(set(group_entity['packages'] + groups[group_name]))
+ else:
+ group_entity['packages'] = groups[group_name]
+ ckan.group_entity_put(group_entity)
+ except CkanApiError, e:
+ if ckan.last_status == 404:
+ print "group "+group_name+" does not exist, creating"
+ group_entity = {
+ 'name': group_url,
+ 'title': group_name,
+ 'description': group_name,
+ 'packages': groups[group_name]
+ }
+ #print group_entity
+ ckan.group_register_post(group_entity)
+ elif ckan.last_status == 409:
+ print "group already exists"
+ else:
+ raise LoaderError('Unexpected status %s adding to group under \'%s\': %r' % (
+ ckan.last_status, pkg_name, e.args))
+
--- a/documents/datagov-export.py
+++ b/documents/datagov-export.py
@@ -1,19 +1,92 @@
+# coding=utf-8
import ckanclient
import couchdb
from ckanclient import CkanApiError
import re
import html2text # aaronsw :(
+import ckanapi # https://github.com/open-data/ckanapi
+import scrape
+import datetime, os, hashlib
class LoaderError(Exception):
pass
+
+def add_package_resource_cachedurl(ckan, package_name, url, name, format, size, **kwargs):
+ # fileupload
+ ts = datetime.datetime.isoformat(datetime.datetime.now()).replace(':', '').split('.')[0]
+
+ file_key = os.path.join(ts, name)
+
+ auth_dict = ckan.storage_auth_get('/form/' + file_key, {})
+
+ fields = [(kv['name'].encode('ascii'), kv['value'].encode('ascii'))
+ for kv in auth_dict['fields']]
+ (url, mime_type, content) = scrape.fetchURL(scrape.docsdb,
+ url, "dataset_resource", "AGIMO", False)
+
+ files = [('file', os.path.basename(file_key), content)]
+
+ errcode, body = ckan._post_multipart(auth_dict['action'].encode('ascii'), fields, files)
+
+ if errcode == 200:
+ file_metadata = ckan.storage_metadata_get(file_key)
+ (url, msg) = file_metadata['_location'], ''
+ else:
+ (url, msg) = '', body
+ # fileupload done
+
+ if url == '':
+ raise CkanApiError(msg)
+ m = hashlib.sha1(msg)
+ #todo mime-type dectection based on content
+ r = dict(name=name,
+ mimetype=mime_type,
+ hash=m.hexdigest(), size=size, url=url)
+
+ r.update(kwargs)
+ if not r.has_key('name'): r['name'] = url
+
+ p = ckan.package_entity_get(package_name)
+ p['resources'].append(r)
+ return ckan.package_entity_put(p)
+
+
# Instantiate the CKAN client.
-#ckan = ckanclient.CkanClient(base_location='http://localhost:5000/api', api_key='b47b24cd-591d-40c1-8677-d73101d56d1b')
-ckan = ckanclient.CkanClient(base_location='http://data.disclosurelo.gs/api',
- api_key='482a9dd2-a976-4adf-ac77-d71d92a98a52')
+api_key = 'ff34526e-f794-4068-8235-fcbba38cd8bc'
+server = 'data.disclosurelo.gs'
+
+ckan = ckanclient.CkanClient(base_location='http://' + server + '/api',
+ api_key=api_key)
+ckandirect = ckanapi.RemoteCKAN('http://' + server, api_key=api_key)
couch = couchdb.Server('http://127.0.0.1:5984/')
#couch = couchdb.Server('http://192.168.1.113:5984/')
+
+import urllib
+import urlparse
+
+
+def url_fix(s, charset='utf-8'):
+ """Sometimes you get an URL by a user that just isn't a real
+ URL because it contains unsafe characters like ' ' and so on. This
+ function can fix some of the problems in a similar way browsers
+ handle data entered by the user:
+
+ >>> url_fix(u'http://de.wikipedia.org/wiki/Elf (Begriffsklärung)')
+ 'http://de.wikipedia.org/wiki/Elf%20%28Begriffskl%C3%A4rung%29'
+
+ :param charset: The target charset for the URL if the url was
+ given as unicode string.
+ """
+ if isinstance(s, unicode):
+ s = s.encode(charset, 'ignore')
+ if not urlparse.urlparse(s).scheme:
+ s = "http://" + s
+ scheme, netloc, path, qs, anchor = urlparse.urlsplit(s)
+ path = urllib.quote(path, '/%')
+ qs = urllib.quote_plus(qs, ':&=')
+ return urlparse.urlunsplit((scheme, netloc, path, qs, anchor))
# http://code.activestate.com/recipes/578019-bytes-to-human-human-to-bytes-converter/
SYMBOLS = {
@@ -24,6 +97,7 @@
'iec_ext': ('byte', 'kibi', 'mebi', 'gibi', 'tebi', 'pebi', 'exbi',
'zebi', 'yobi'),
}
+
def human2bytes(s):
"""
@@ -53,6 +127,9 @@
...
ValueError: can't interpret '12 foo'
"""
+ if s == None:
+ return 0
+ s = s.replace(',', '')
init = s
num = ""
while s and s[0:1].isdigit() or s[0:1] == '.':
@@ -91,8 +168,6 @@
def name_munge(input_name):
return munge(input_name.replace(' ', '').replace('.', '_').replace('&', 'and'))
- #[:100]
- #return input_name.replace(' ', '').replace('.', '_').replace('&', 'and')
def get_licence_id(licencename):
@@ -113,110 +188,142 @@
raise Exception(licencename + " not found");
return map[licencename];
+
+gooddata = ["afl-in-victoria", "annual-budget-initiatives-by-suburb-brisbane-city-council"]
+#athletics-in-victoria-gfyl,bicycle-racks-mosman-municipal-council,boat-ramps-brisbane-city-council,brisbane-access-ratings-database,bus-stops-brisbane-city-council,cemeteries-brisbane-city-council,cfa-locations,citycycle-stations-brisbane-city-council,community-gardens-brisbane-city-council,community-halls-brisbane-city-council,cooking-classes-gfyl,court-locations-victoria,customer-service-centres-brisbane-city-council,dance-in-victoria-gfyl,disability-activity-gfyl,dog-parks-brisbane-city-council,ferry-terminals-brisbane-city-council,fishing-club-in-victoria-gfyl,fitness-centres-in-victoria-gfyl,gardens-reserves-gfyl,golf-courses-brisbane-city-council,gymnastics-in-victoria-gfyl,historic-cemeteries-brisbane-city-council,ice-skating-centres-gfyl,immunisation-clinics-brisbane-city-council,libraries-brisbane-city-council,licenced-venues-victoria,lifesaving-locations-victoria,loading-zones-brisbane-city-council,major-projects-victoria,markets-in-victoria,martial-arts-in-victoria-gfyl,melbourne-water-use-by-postcode,members-of-parliament-both-houses-nsw,members-of-the-legislative-assembly-nsw,members-of-the-legislative-council-nsw,mfb-locations-vic,ministers-of-the-nsw-parliament,mosman-local-government-area,mosman-rider-route,mosman-wwii-honour-roll,neighbourhood-houses-gfyl,news-feeds-mosman-municipal-council,off-street-car-parks-mosman-municipal-council,orienteering-clubs-gfyl,parking-meter-areas-brisbane-city-council,parks-and-reserves-mosman-municipal-council,parks-brisbane-city-council,personal-training-gfyl,picnic-areas-brisbane-city-council,playgrounds-brisbane-city-council,playgrounds-mosman-municipal-council,police-region-crime-statistics-victoria,police-service-area-crime-statistics-victoria,pony-clubs-in-victoria-gfyl,prison-locations-victoria,public-amenities-maintained-by-mosman-council,public-art-brisbane-city-council,public-internet-locations-vic,public-toilets-brisbane-city-council,racecourse-locations-victoria,recent-development-applications-mosman-municipal-council,recreation-groups-gfyl,recreational-fishing-spots,regional-business-centres-brisbane-city-council,reports-of-swooping-birds-mosman-municipal-council,restricted-parking-areas-brisbane-city-council,rollerskating-centres-in-victoria-gfyl,sailing-clubs-gfyl,school-locations-victoria,shadow-ministers-of-the-nsw-parliament,skate-parks-gfyl,sporting-clubs-and-organisations-gfyl,stakeboard-parks-brisbane-city-council,state-bodies-gfyl,street-names-brisbane-city-council,suburbs-and-adjoining-suburbs-brisbane-city-council,swimming-pools-brisbane-city-council,swimming-pools-gfyl,tennis-courts-brisbane-city-council,top-40-book-club-reads-brisbane-city-council,tracks-and-trails-gfyl,triathlon-clubs-gfyl,urban-water-restrictions-victoria,veterinary-services-in-mosman,victorian-microbreweries,volunteering-centres-services-and-groups-victoria,walking-groups-gfyl,ward-offices-brisbane-city-council,waste-collection-days-brisbane-city-council,waste-transfer-stations-brisbane-city-council,water-consumption-in-melbourne,water-sports-in-victoria-gfyl,wifi-hot-spots-brisbane-city-council,yoga-pilates-and-tai-chi-in-victoria-gfyl,2809cycling-in-new-south-wales-what-the-data-tells-us2809-and-related-data,act-barbecue-bbq-locations,act-tafe-locations,ausindustry-locations,austender-contract-notice-export,austender-contract-notice-export,austender-contract-notice-export,austender-contract-notice-export,austender-contract-notice-export,austender-contract-notice-export,austender-contract-notice-export,austender-contract-notice-export,australian-gas-light-company-maps,australian-gas-light-company-maps,australian-ports,australian-public-service-statistical-bulletin-2011-12,australian-public-service-statistical-bulletin-snapshot-at-december-31-2011,australian-public-service-statistical-bulletin-tables-0910,austrics-timetable-set,capital-works-call-tender-schedule,collection-item-usage-state-library-of-victoria,country-and-commodity-trade-data-spreadsheet,country-and-commodity-trade-data-spreadsheet-2,country-by-level-of-processing-trade-data-spreadsheet,crime-incident-type-and-frequency-by-capital-city-and-nationally,csiro-locations,data-from-the-oaic-public-sector-information-survey-2012,data-from-the-oaic-public-sector-information-survey-2012,data-from-the-oaic-public-sector-information-survey-2012,department-of-finance-and-deregulation-office-locations,digitised-maps,diisr-division-locations-excluding-ausindustry-enterprise-connect-and-nmi,diisr-locations,diisr-portfolio-agency-locations-excluding-csiro,distance-to-legal-service-providers-from-disadvantaged-suburbs,enterprise-connect-locations,fire-insurance-maps-sydney-block-plans-1919-1940,fire-insurance-maps-sydney-block-plans-1919-1940,first-fleet-collection,first-fleet-collection,first-fleet-maps,first-fleet-maps,freedom-of-information-annual-estimated-costs-and-staff-time-statistical-data-2011-12,freedom-of-information-quarterly-request-and-review-statistical-data-2011-12,freedom-of-information-requests-estimated-costs-and-charges-collected-1982-83-to-2011-12,higher-education-course-completions,higher-education-enrolments,historical-australian-government-contract-data,historical-australian-government-contract-data,historical-australian-government-contract-data,historical-australian-government-contract-data,historical-australian-government-contract-data,historical-australian-government-contract-data,historical-australian-government-contract-data,historical-australian-government-contract-data,historical-australian-government-contract-data,historical-australian-government-contract-data,historical-australian-government-contract-data,historical-australian-government-contract-data,historical-australian-government-contract-data,historical-australian-government-contract-data,historical-australian-government-contract-data,journey-planner-data-nt,library-catalogue-search-terms-state-library-of-victoria,location-of-act-schools,location-of-centrelink-offices,location-of-european-wasps-nests,location-of-lawyers-and-legal-service-providers-by-town,location-of-legal-assistance-service-providers,location-of-medicare-offices,location-of-medicare-offices,maps-of-the-southern-hemisphere-16th-18th-centuries,maps-of-the-southern-hemisphere-16th-18th-centuries,music-queensland,national-measurement-institute-locations,new-south-wales-officers-and-men-of-the-australian-imperial-force-a-i-f-and-the-australian-naval-for,new-south-wales-officers-and-men-of-the-australian-imperial-force-a-i-f-and-the-australian-naval-for,photographs-of-nsw-life-pre-1955,photographs-of-nsw-life-pre-1955,photographs-of-sydney-before-1885,photographs-of-sydney-before-1885,picture-queensland,plgr-28093-playgrounds-act,police-station-locations,queensland-public-libraries,rare-printed-books,rare-printed-books,real-estate-maps,regional-australia-funding-projects,sa-memory-state-library-of-south-australia,search-engine-terms-state-library-of-victoria,south-australian-photographs-state-library-of-south-australia,south-australian-sheet-music-state-library-of-south-australia,sydney-bond-store-maps-1894,sydney-bond-store-maps-1894,sydney-maps-1917,sydney-maps-1917,tafe-institute-locations-victoria,tafe-sa-campus-locations,tolt-public-toilets-act,victorian-public-library-branches-state-library-of-victoria,western-australia-public-library-network,world-war-one-photographs-by-frank-hurley,world-war-one-photographs-by-frank-hurley,citycat-timetables-brisbane-city-council,cityferry-timetables-brisbane-city-council,cost-of-salinity-to-local-infrastructure-1996-97-summary-of-component-costs-of-salinity-by-reporting,cost-of-salinity-to-local-infrastructure-1996-97-summary-of-component-costs-of-salinity-by-reporting,downstream-cost-calculator-model-and-data-for-199697-or-2001-prices,economics-of-australian-soil-conditions-199697-limiting-factor-or-relative-yield-min-of-ry_salt2000-,geographical-names-register-gnr-of-nsw,victorian-dryland-salinity-assessment-2000-d01cac_ramsar_final-xls,victorian-dryland-salinity-assessment-2000-d02cac_fauna_final-xls,victorian-dryland-salinity-assessment-2000-d03cac_fauna_dist_final-xls,victorian-dryland-salinity-assessment-2000-dc04cac_hydrol_final-xls,victorian-dryland-salinity-assessment-2000-dc05cac_wetland_final-xls,victorian-dryland-salinity-assessment-2000-dc06cac_util_final-xls,victorian-dryland-salinity-assessment-2000-dc07cac_road_final-xls,victorian-dryland-salinity-assessment-2000-dc08cac_towns_final-xls,victorian-dryland-salinity-assessment-2000-dc09cac_flora_final-xls,victorian-dryland-salinity-assessment-2000-dc10cac_flora_dist_final-xls,victorian-dryland-salinity-assessment-2000-dc12cac_infrastructure-xls,victorian-dryland-salinity-assessment-2000-dc13cac_natural_envt-xls,victorian-dryland-salinity-assessment-2000-dc14cac_agriculture-xls,victorian-dryland-salinity-assessment-2000-dc16cac_agric_cost-xls,victorian-dryland-salinity-assessment-2000-dc17cac_shallow_wt-xls,victorian-dryland-salinity-assessment-2000-dc18cac_agric_cost_time-xls,victorian-dryland-salinity-assessment-2000-dc21cac_water_resources_new-xls,victorian-dryland-salinity-assessment-2000-dc22cac_risk-xls,licensed-broadcasting-transmitter-data,nsw-crime-data,recorded-crime-dataset-nsw,crime-statistics-in-nsw-by-month,2001-02-to-2007-08-local-government-survey-victoria,2009-green-light-report,annual-statistical-reports-fire-brigades-nsw-200304,annual-statistical-reports-fire-brigades-nsw-200405,annual-statistical-reports-fire-brigades-nsw-200506,annual-statistical-reports-fire-brigades-nsw-200607,arts-on-the-map,assets-and-liabilities-of-australian-located-operations,assets-of-australian-located-operations,assets-of-australian-located-operations-by-country,assets-of-financial-institutions,back-issues-of-monthly-banking-statistics,banks-assets,banks-consolidated-group-capital,banks-consolidated-group-impaired-assets,banks-consolidated-group-off-balance-sheet-business,banks-liabilities,building-societies-selected-assets-and-liabilities,byteback2842-locations-vic,cash-management-trusts,city-of-melbourne-street-furniture-database,community-services-nsw,consolidated-exposures-immediate-and-ultimate-risk-basis,consolidated-exposures-immediate-risk-basis-foreign-claims-by-country,consolidated-exposures-immediate-risk-basis-international-claims-by-country,consolidated-exposures-ultimate-risk-basis,consolidated-exposures-ultimate-risk-basis-foreign-claims-by-country,cosolidated-exposures-immediate-risk-basis,credit-unions-selected-assets-and-liabilities,daily-net-foreign-exchange-transactions,detox-your-home,education-national-assessment-program-literacy-and-numeracy-nsw,employment-data-by-nsw-regions,excise-beer-clearance-data-updated-each-month-beer-clearance-summary-data,finance-companies-and-general-financiers-selected-assets-and-liabilities,foreign-exchange-transactions-and-holdings-of-official-reserve-assets,half-yearly-life-insurance-bulletin-december-2010,health-behaviours-in-nsw,international-liabilities-by-country-of-the-australian-located-operations-of-banks-and-rfcs,liabilities-and-assets-monthly,liabilities-and-assets-weekly,liabilities-of-australian-located-operations,life-insurance-offices-statutory-funds,managed-funds,monetary-policy-changes,money-market-corporations-selected-assets-and-liabilities,monthly-airport-traffic-data-for-top-ten-airports-january-1985-to-december-2008,monthly-banking-statistics-april-2011,monthly-banking-statistics-june-2011,monthly-banking-statistics-may-2011,open-market-operations-2009-to-current,projected-households-vic-rvic-msd-2006-2056,projected-population-by-age-and-sex-vic-rvic-msd-2006-2056,public-unit-trust,quarterly-bank-performance-statistics,quarterly-general-insurance-performance-statistics-march-2011,quarterly-superannuation-performance-march-2011,recorded-crime-dataset-nsw,residential-land-bulletin,resourcesmart-retailers,resourcesmart-retailers-vic,road-fatalities-nsw,securitisation-vehicles,selected-asset-and-liabilities-of-the-private-non-financial-sectors,seperannuation-funds-outside-life-offices,solar-report-vic,towns-in-time-victoria,vif2008-projected-population-by-5-year-age-groups-and-sex-sla-lga-ssd-sd-2006-2026,vif2008-projected-population-totals-and-components-vic-rvic-msd-2006-2056,vif2008-projected-population-totals-sla-lga-ssd-sd-2006-2026,arts-festivals-victoria,arts-organisations-victoria,arts-spaces-and-places-victoria,ausgrid-average-electricity-use,collecting-institutions-victoria,indigenous-arts-organisations-victoria,latest-coastal-weather-observations-for-coolangatta-qld,top-10-fiction-books-brisbane-city-council];
+
+
docsdb = couch['disclosr-documents']
if __name__ == "__main__":
+ orgs_list = []
+ orgs_ids = {}
for doc in docsdb.view('app/datasets'):
+ print " --- "
print doc.id
+
if doc.value['url'] != "http://data.gov.au/data/" and doc.value['agencyID'] != "qld":
+
+
# Collect the package metadata.
- pkg_name = doc.value['url'].replace("http://data.gov.au/dataset/",'').replace('/','');
- tags = []
- if doc.value['agencyID'] == "AGIMO":
- if len(doc.value['metadata']["Keywords / Tags"]) > 0:
- if hasattr(doc.value['metadata']["Keywords / Tags"], '__iter__'):
- tags = tags + doc.value['metadata']["Keywords / Tags"]
+ pkg_name = filter(lambda x: x in '0123456789abcdefghijklmnopqrstuvwxyz-_',
+ doc.value['url'].replace("http://data.gov.au/dataset/", '').replace('/', '')[:100]);
+ print pkg_name
+ if pkg_name in gooddata:
+
+ #add to or create organization using direct API
+ agency = doc.value['metadata']["Agency"]
+ if agency == "APS":
+ agency = "Australian Public Service Commission"
+ if agency == "Shared Services, Treasury Directorate":
+ agency = "Shared Services Procurement, Treasury Directorate"
+ if agency == "Treasury - Shared Services":
+ agency = "Shared Services Procurement, Treasury Directorate"
+ if agency == "Territory and Municipal Services (TAMS)":
+ agency = "Territory and Municipal Services Directorate"
+ if agency == "State Library of NSW":
+ agency = "State Library of New South Wales"
+ org_name = name_munge(agency[:100])
+ if org_name not in orgs_list:
+ orgs_list = ckandirect.action.organization_list()['result']
+ #print orgs_list
+ if org_name not in orgs_list:
+ try:
+ print "org not found, creating " + org_name
+ ckandirect.action.organization_create(name=org_name, title=agency,
+ description=agency)
+ orgs_list.append(org_name)
+ except ckanapi.ValidationError, e:
+ print e
+ raise LoaderError('Unexpected status')
else:
- tags = tags + [doc.value['metadata']["Keywords / Tags"]]
- if 'data.gov.au Category' in doc.value['metadata'].keys() and len(doc.value['metadata']['data.gov.au Category']) > 0:
- if hasattr(doc.value['metadata']['data.gov.au Category'], '__iter__'):
- tags = tags + doc.value['metadata']['data.gov.au Category']
+ print "org found, adding dataset to " + org_name
+
+ # cache org names -> id mapping
+ if org_name not in orgs_ids:
+ org = ckandirect.action.organization_show(id=org_name)
+ orgs_ids[org_name] = org["result"]["id"]
+ org_id = orgs_ids[org_name]
+ print "org id is " + org_id
+ tags = []
+ creator = doc.value['metadata']["DCTERMS.Creator"]
+ if doc.value['agencyID'] == "AGIMO":
+ if len(doc.value['metadata']["Keywords / Tags"]) > 0:
+ if hasattr(doc.value['metadata']["Keywords / Tags"], '__iter__'):
+ tags = tags + doc.value['metadata']["Keywords / Tags"]
+ else:
+ tags = tags + [doc.value['metadata']["Keywords / Tags"]]
+
+ tags = [re.sub('[^a-zA-Z0-9-_.]', '', tag.replace('&', 'and')).lower() for tag in tags if tag]
+ #print tags
+ extras = []
+
+ for extra_key in doc.value['metadata'].keys():
+ if extra_key not in ["Description", "Content-Language", "DCTERMS.Description",
+ "Keywords / Tags",
+ "data.gov.au Category", "Download", "Permalink", "DCTERMS.Identifier"]:
+ if doc.value['metadata'][extra_key] != None and doc.value['metadata'][extra_key] != "":
+ extras.append([extra_key, doc.value['metadata'][extra_key]])
+
+ package_entity = {
+ 'name': pkg_name,
+ 'title': doc.value['metadata']['DCTERMS.Title'],
+ 'url': doc.value['metadata']['DCTERMS.Source.URI'],
+ 'tags': tags, #tags are mandatory?
+ 'author': creator,
+ 'maintainer': creator,
+ 'licence_id': get_licence_id(doc.value['metadata']['DCTERMS.License']),
+ 'notes': html2text.html2text(doc.value['metadata']['Description']),
+ 'owner_org': org_id,
+ 'extras': extras
+ }
+
+ try:
+ #print package_entity
+ ckan.package_register_post(package_entity)
+ except CkanApiError, e:
+ if ckan.last_message == "{\"name\": [\"That URL is already in use.\"]}":
+ print "package already exists"
else:
- tags = tags + [doc.value['metadata']['data.gov.au Category']]
- tags = [re.sub('[^a-zA-Z0-9-_.]', '', tag.replace('&', 'and')).lower() for tag in tags if tag]
- print tags
- package_entity = {
- 'name': pkg_name,
- 'title': doc.value['metadata']['DCTERMS.Title'],
- 'url': doc.value['metadata']['DCTERMS.Source.URI'],
- 'tags': tags, #tags are mandatory?
- 'author': doc.value['metadata']["DCTERMS.Creator"],
- 'maintainer': doc.value['metadata']["DCTERMS.Creator"],
- 'licence_id': get_licence_id(doc.value['metadata']['DCTERMS.License']),
- 'notes': html2text.html2text(doc.value['metadata']['Description']),
- }
- if doc.value['agencyID'] == "qld":
- package_entity = doc.value['metadata']
-
- try:
- print package_entity
- ckan.package_register_post(package_entity)
- except CkanApiError, e:
- if ckan.last_status == 409:
- print "package already exists"
- else:
- raise LoaderError('Unexpected status %s checking for package under \'%s\': %r' % (
- ckan.last_status, pkg_name, e.args))
-
-
- #add to group
-
- group_name = name_munge(doc.value['metadata']["Agency"][:100])
- try:
- print ckan.group_entity_get(group_name)
-
- # Update the group details
- group_entity = ckan.last_message
- print "group exists"
- if 'packages' in group_entity.keys():
- group_entity['packages'] = list(set(group_entity['packages'] + [pkg_name]))
- else:
- group_entity['packages'] = [pkg_name]
- ckan.group_entity_put(group_entity)
- except CkanApiError, e:
- if ckan.last_status == 404:
- print "group does not exist, creating"
- group_entity = {
- 'name': group_name,
- 'title': doc.value['metadata']["Agency"],
- 'description': doc.value['metadata']["Agency"],
- 'packages': [pkg_name],
- # 'type': "organization" # not allowed via API, use database query
- # update "group" set type = 'organization';
- }
- print group_entity
- ckan.group_register_post(group_entity)
- elif ckan.last_status == 409:
- print "group already exists"
- else:
- raise LoaderError('Unexpected status %s adding to group under \'%s\': %r' % (
- ckan.last_status, pkg_name, e.args))
- if 'Download' in doc.value['metadata'].keys():
- try:
- pkg = ckan.package_entity_get(pkg_name)
- resources = pkg.get('resources', [])
- if len(resources) < len(doc.value['metadata']['Download']):
- for resource in doc.value['metadata']['Download']:
-
- # http://docs.ckan.org/en/ckan-1.7/domain-model-resource.html
- # (KML/KMZ) / (Shapefile) /(Other)
- format = "plain"
- if resource['format'] == '(XML)':
- format = 'xml'
- if resource['format'] == '(CSV/XLS)':
- format = 'csv'
- name = resource['href']
- if 'name' in resource.keys():
- name = resource['name']
- print resource
- ckan.add_package_resource(pkg_name, resource['href'], name=name, resource_type='data',
- format=format, size=human2bytes(resource['size'].replace(',', '')))
- else:
- print "resources already exist"
- except CkanApiError, e:
- if ckan.last_status == 404:
- print "parent dataset does not exist"
- else:
+ print ckan.last_message
raise LoaderError('Unexpected status %s checking for package under \'%s\': %r' % (
ckan.last_status, pkg_name, e.args))
-
+ pkg = ckan.package_entity_get(pkg_name)
+
+
+ # add resources (downloadable data files)
+ if 'Download' in doc.value['metadata'].keys():
+ try:
+
+ resources = pkg.get('resources', [])
+ if len(resources) < len(doc.value['metadata']['Download']):
+ for resource in doc.value['metadata']['Download']:
+
+ # http://docs.ckan.org/en/ckan-1.7/domain-model-resource.html
+ # (KML/KMZ) / (Shapefile) /(Other)
+ format = "plain"
+ if resource['format'] == '(XML)':
+ format = 'xml'
+ if resource['format'] == '(CSV/XLS)':
+ format = 'csv'
+ if resource['format'] == '(Shapefile)':
+ format = 'shp'
+ if resource['format'] == '(KML/KMZ)':
+ format = 'kml'
+ name = resource['href']
+ if 'name' in resource.keys():
+ name = resource['name']
+ print resource
+ add_package_resource_cachedurl(ckan, pkg_name, url_fix(resource['href']), name,
+ format,
+ human2bytes(resource.get('size', '0B')),
+ resource_type='data')
+ else:
+ print "resources already exist"
+ except CkanApiError, e:
+ if ckan.last_status == 404:
+ print "parent dataset does not exist"
+ else:
+ raise LoaderError('Unexpected status %s checking for package under \'%s\': %r' % (
+ ckan.last_status, pkg_name, e.args))
+
--- /dev/null
+++ b/documents/datagov-resourcereport.py
@@ -1,1 +1,81 @@
+import couchdb
+couch = couchdb.Server('http://127.0.0.1:5984/')
+#couch = couchdb.Server('http://192.168.1.113:5984/')
+import urllib
+import urlparse
+import httplib2
+import httplib
+import csv
+
+
+def url_fix(s, charset='utf-8'):
+ """Sometimes you get an URL by a user that just isn't a real
+ URL because it contains unsafe characters like ' ' and so on. This
+ function can fix some of the problems in a similar way browsers
+ handle data entered by the user:
+
+ :param charset: The target charset for the URL if the url was
+ given as unicode string.
+ """
+ if isinstance(s, unicode):
+ s = s.encode(charset, 'ignore')
+ if not urlparse.urlparse(s).scheme:
+ s = "http://"+s
+ scheme, netloc, path, qs, anchor = urlparse.urlsplit(s)
+ path = urllib.quote(path, '/%')
+ qs = urllib.quote_plus(qs, ':&=')
+ return urlparse.urlunsplit((scheme, netloc, path, qs, anchor))
+
+# http://code.activestate.com/recipes/578019-bytes-to-human-human-to-bytes-converter/
+SYMBOLS = {
+ 'customary': ('B', 'KB', 'MB', 'GB', 'T', 'P', 'E', 'Z', 'Y'),
+ 'customary_ext': ('byte', 'kilo', 'mega', 'giga', 'tera', 'peta', 'exa',
+ 'zetta', 'iotta'),
+ 'iec': ('Bi', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi'),
+ 'iec_ext': ('byte', 'kibi', 'mebi', 'gibi', 'tebi', 'pebi', 'exbi',
+ 'zebi', 'yobi'),
+}
+
+
+docsdb = couch['disclosr-documents']
+out = csv.writer(open("output.csv","w"), delimiter=',',quoting=csv.QUOTE_ALL)
+if __name__ == "__main__":
+ for doc in docsdb.view('app/datasets'):
+ if doc.value['url'] != "http://data.gov.au/data/" and doc.value['agencyID'] != "qld":
+ # Collect the package metadata.
+ pkg_name = filter(lambda x: x in '0123456789abcdefghijklmnopqrstuvwxyz-_',
+ doc.value['url'].replace("http://data.gov.au/dataset/", '').replace('/', '')[:100]);
+ if 'Download' in doc.value['metadata'].keys() and len(doc.value['metadata']['Download']) > 0:
+ for resource in doc.value['metadata']['Download']:
+ # http://docs.ckan.org/en/ckan-1.7/domain-model-resource.html
+ # (KML/KMZ) / (Shapefile) /(Other)
+ format = "plain"
+ if resource['format'] == '(XML)':
+ format = 'xml'
+ if resource['format'] == '(CSV/XLS)':
+ format = 'csv'
+ if resource['format'] == '(Shapefile)':
+ format = 'shp'
+ if resource['format'] == '(KML/KMZ)':
+ format = 'kml'
+ name = resource['href']
+ if 'name' in resource.keys():
+ name = resource['name']
+ if resource['href'].startswith("ftp"):
+ out.writerow([pkg_name, url_fix(resource['href']), name,format, "ftp", ""])
+ else:
+ try:
+ h = httplib2.Http(disable_ssl_certificate_validation=True)
+ resp = h.request(url_fix(resource['href']), 'HEAD')
+ content_type = resp[0]['content-type'] if 'content-type' in resp[0].keys() else ""
+ out.writerow([pkg_name.encode('ascii', 'ignore'), url_fix(resource['href']).encode('ascii', 'ignore'), name.encode('ascii', 'ignore'),format, resp[0]['status'], content_type])
+ except httplib2.ServerNotFoundError:
+ out.writerow([pkg_name.encode('ascii', 'ignore'), url_fix(resource['href']).encode('ascii', 'ignore'), name.encode('ascii', 'ignore'),format, "500","badurl"])
+ except httplib.InvalidURL:
+ out.writerow([pkg_name.encode('ascii', 'ignore'), url_fix(resource['href']).encode('ascii', 'ignore'), name.encode('ascii', 'ignore'),format, "500","badurl"])
+ except httplib2.RelativeURIError:
+ out.writerow([pkg_name.encode('ascii', 'ignore'), url_fix(resource['href']).encode('ascii', 'ignore'), name.encode('ascii', 'ignore'),format, "500","badurl"])
+ else:
+ out.writerow([pkg_name.encode('ascii', 'ignore')])
+
--- a/documents/datagov.py
+++ b/documents/datagov.py
@@ -39,7 +39,9 @@
link = item.find("a")
format = item.find(property="dc:format")
linkobj = {"href":link['href'].replace("/bye?","").strip(),
- "format": format.string.strip(), "size": format.next_sibling.string.strip()}
+ "format": format.string.strip()}
+ if format.next_sibling.string != None:
+ linkobj["size"] = format.next_sibling.string.strip()
if link.string != None:
linkobj["name"] = link.string.strip()
doc['metadata'][last_title].append(linkobj)
--- a/documents/template.inc.php
+++ b/documents/template.inc.php
@@ -58,11 +58,11 @@
<div class="navbar navbar-inverse navbar-fixed-top">
<div class="navbar-inner">
<div class="container-fluid">
- <a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
+ <!--<a class="btn btn-navbar" data-toggle="collapse" data-target=".nav-collapse">
<span class="icon-bar"></span>
<span class="icon-bar"></span>
<span class="icon-bar"></span>
- </a>
+ </a> -->
<a class="brand" href="#">Australian Disclosure Logs</a>
<div class="nav-collapse collapse">
@@ -79,6 +79,7 @@
<li><a href="agency.php">By Agency</a></li>
<li><a href="date.php">By Date</a></li>
<li><a href="disclogsList.php">List of Disclosure Logs</a></li>
+ <li><a href="charts.php">Charts</a></li>
<li><a href="about.php">About</a></li>
</ul>
--- /dev/null
+++ b/employees-fail.php
@@ -1,1 +1,134 @@
+<?php
+include_once('include/common.inc.php');
+include_header('Charts');
+$db = $server->get_db('disclosr-agencies');
+?>
+<div class="foundation-header">
+ <h1><a href="about.php">Charts</a></h1>
+ <h4 class="subheader">Lorem ipsum.</h4>
+</div>
+<div id="scores" style="width:900px;height:500px;"></div>
+<script id="source">
+ window.onload = function() {
+ $(document).ready(function() {
+ var d1 = [];
+ var scorelabels = [];
+ <?php
+ try {
+ $rows = $db->get_view("app", "scoreHas?group=true", null, true)->rows;
+
+ $dataValues = Array();
+ foreach ($rows as $row) {
+ $dataValues[$row->value] = $row->key;
+ }
+ $i = 0;
+ ksort($dataValues);
+ foreach ($dataValues as $value => $key) {
+
+ echo " d1.push([$i, $value]);" . PHP_EOL;
+ echo " scorelabels.push('$key');" . PHP_EOL;
+ $i++;
+ }
+ } catch (SetteeRestClientException $e) {
+ setteErrorHandler($e);
+ }
+ ?>
+ function scoretrackformatter(obj) {
+ if (scorelabels[Math.floor(obj.x)]) {
+ return (scorelabels[Math.floor(obj.x)])+"="+obj.y;
+
+ } else {
+ return "";
+ }
+ }
+ function scoretickformatter(val, axis) {
+ if (scorelabels[Math.floor(val)]) {
+ return '<p style="margin-top:8em;-webkit-transform:rotate(-90deg);">'+(scorelabels[Math.floor(val)])+"</b>";
+
+ } else {
+ return "";
+ }
+ }
+ Flotr.draw(document.getElementById("scores"), [ {data: d1}], {
+ HtmlText: true,
+ bars : {
+ show : true
+ },
+ mouse : {
+ track : true,
+ relative : true,
+ trackFormatter: scoretrackformatter
+ },yaxis: {
+ autoscaling: true
+ },
+ xaxis: {
+ autoscaling: true,
+ minorTickFreq: 0.6,
+ noTicks : scorelabels.length,
+ tickFormatter: scoretickformatter
+ }
+ });
+
+
+
+
+
+
+
+
+<div id="employees" style="width:1000px;height:900px;"></div>
+var emplabels = [];
+function emptrackformatter(obj) {
+
+ return (obj.series.label)+" = "+obj.y+" in "+emplabels[Math.floor(obj.x)];
+
+ }
+ function emptickformatter(val, axis) {
+ if (emplabels[Math.floor(val)]) {
+ return '<p style="margin-top:8em;-webkit-transform:rotate(-90deg);">'+(emplabels[Math.floor(val)])+"</b>";
+
+ } else {
+ return "";
+ }
+ }
+function onDataReceived(series) {
+ emplabels = series.labels;
+ Flotr.draw(document.getElementById("employees"), series.data, {
+ mouse : {
+ track : true,
+ relative : true,
+ trackFormatter: emptrackformatter
+ },yaxis: {
+ max: 10000,
+ scaling: 'logarithmic'
+ },
+ xaxis: {
+ minorTickFreq: 1,
+ noTicks: emplabels.length,
+ showMinorLabels: true,
+ tickFormatter: emptickformatter
+ },
+ legend: {
+ show: false
+ }
+ });
+ }
+
+ $.ajax({
+ url: "admin/exportEmployees.csv.php?format=json",
+ method: 'GET',
+ dataType: 'json',
+ success: onDataReceived
+ });
+
+
+ });
+ };
+
+</script>
+
+<?php
+include_footer();
+?>
+
--- a/getAgency.php
+++ b/getAgency.php
@@ -2,11 +2,12 @@
include_once('include/common.inc.php');
-function displayValue($key, $value, $mode) {
+function displayValue($key, $value, $mode)
+{
global $db, $schemas;
- $ignoreKeys = Array("metadata" ,"metaTags", "statistics","rtkURLs","rtkDescriptions");
+ $ignoreKeys = Array("metadata", "metaTags", "statistics", "rtkURLs", "rtkDescriptions");
if ($mode == "view") {
- if (strpos($key, "_") === 0 || in_array($key,$ignoreKeys))
+ if (strpos($key, "_") === 0 || in_array($key, $ignoreKeys))
return;
echo "<tr>";
@@ -22,7 +23,8 @@
echo "<li ";
if (isset($schemas['agency']["properties"][$key]['x-property'])) {
echo ' property="' . $schemas['agency']["properties"][$key]['x-property'] . '" ';
- } if (isset($schemas['agency']["properties"][$key]['x-itemprop'])) {
+ }
+ if (isset($schemas['agency']["properties"][$key]['x-itemprop'])) {
echo ' itemprop="' . $schemas['agency']["properties"][$key]['x-itemprop'] . '" ';
}
echo " >";
@@ -62,7 +64,7 @@
</div>";
} else {
if (strpos($key, "_") === 0) {
- echo"<input type='hidden' id='$key' name='$key' value='$value'/>";
+ echo "<input type='hidden' id='$key' name='$key' value='$value'/>";
} else if ($key == "parentOrg") {
echo "<label for='$key'>$key</label><select id='$key' name='$key'><option value=''> Select... </option>";
$rows = $db->get_view("app", "byDeptStateName")->rows;
@@ -70,7 +72,7 @@
foreach ($rows as $row) {
echo "<option value='{$row->value}'" . (($row->value == $value) ? "SELECTED" : "") . " >" . str_replace("Department of ", "", $row->key) . "</option>";
}
- echo" </select>";
+ echo " </select>";
} else {
echo "<label>$key</label><input class='input-text' type='text' id='$key' name='$key' value='$value'/>";
if ((strpos($key, "URL") > 0 || $key == 'website') && $value != "") {
@@ -85,7 +87,8 @@
//
}
-function addDefaultFields($row) {
+function addDefaultFields($row)
+{
global $schemas;
$defaultFields = array_keys($schemas['agency']['properties']);
foreach ($defaultFields as $defaultField) {
@@ -119,103 +122,103 @@
// by name = startkey="Ham"&endkey="Ham\ufff0"
// edit?
- $obj = $db->get($_REQUEST['id']);
- include_header(isset($obj->name) ? $obj->name : "");
+$obj = $db->get($_REQUEST['id']);
+include_header(isset($obj->name) ? $obj->name : "");
//print_r($row);
- if (sizeof($_POST) > 0) {
+if (sizeof($_POST) > 0) {
//print_r($_POST);
- foreach ($_POST as $postkey => $postvalue) {
- if ($postvalue == "") {
+ foreach ($_POST as $postkey => $postvalue) {
+ if ($postvalue == "") {
+ unset($_POST[$postkey]);
+ }
+ if (is_array($postvalue)) {
+ if (count($postvalue) == 1 && $postvalue[0] == "") {
unset($_POST[$postkey]);
- }
- if (is_array($postvalue)) {
- if (count($postvalue) == 1 && $postvalue[0] == "") {
- unset($_POST[$postkey]);
- } else {
- foreach ($_POST[$postkey] as $key => &$value) {
- if ($value == "") {
- unset($_POST[$postkey][$key]);
- }
+ } else {
+ foreach ($_POST[$postkey] as $key => &$value) {
+ if ($value == "") {
+ unset($_POST[$postkey][$key]);
}
}
}
}
- if (isset($_POST['_id']) && $db->get_rev($_POST['_id']) == $_POST['_rev']) {
- echo "Edited version was latest version, continue saving";
- $newdoc = $_POST;
- $newdoc['metadata']['lastModified'] = time();
- $obj = $db->save($newdoc);
- } else {
- echo "ALERT doc revised by someone else while editing. Document not saved.";
- }
- }
-
- $mode = "view";
- $rowArray = object_to_array($obj);
- ksort($rowArray);
- if ($mode == "edit") {
- $row = addDefaultFields($rowArray);
+ }
+ if (isset($_POST['_id']) && $db->get_rev($_POST['_id']) == $_POST['_rev']) {
+ echo "Edited version was latest version, continue saving";
+ $newdoc = $_POST;
+ $newdoc['metadata']['lastModified'] = time();
+ $obj = $db->save($newdoc);
} else {
- $row = $rowArray;
- }
-
- if ($mode == "view") {
- echo ' <div class="container-fluid">
+ echo "ALERT doc revised by someone else while editing. Document not saved.";
+ }
+}
+
+$mode = "view";
+$rowArray = object_to_array($obj);
+ksort($rowArray);
+if ($mode == "edit") {
+ $row = addDefaultFields($rowArray);
+} else {
+ $row = $rowArray;
+}
+
+if ($mode == "view") {
+ echo ' <div class="container-fluid">
<div class="row-fluid">
<div class="span3">
<div class="well sidebar-nav">
<ul class="nav nav-list">
<li class="nav-header">Statistics</li>';
-
- if (isset($row['statistics']['employees'])) {
- echo '<div><i class="icon-user" style="float:left"></i><p style="margin-left:16px;">';
- $keys = array_keys($row['statistics']['employees']);
- $lastkey = $keys[count($keys)-1];
- echo $row['statistics']['employees'][$lastkey]['value'].' employees <small>('.$lastkey.')</small>';
- echo '</div>';
- }
- if (isset($row['statistics']['budget'])) {
- echo '<div><i class="icon-shopping-cart" style="float:left"></i><p style="margin-left:16px;">';
- $keys = array_keys($row['statistics']['budget']);
- $lastkey = $keys[count($keys)-1];
- echo "$".number_format(floatval($row['statistics']['budget'][$lastkey]['value'])).' <small>('.$lastkey.' budget)</small>';
- echo '</div>';
- }
+
+ if (isset($row['statistics']['employees'])) {
+ echo '<div><i class="icon-user" style="float:left"></i><p style="margin-left:16px;">';
+ $keys = array_keys($row['statistics']['employees']);
+ $lastkey = $keys[count($keys) - 1];
+ echo $row['statistics']['employees'][$lastkey]['value'] . ' employees <small>(' . $lastkey . ')</small>';
+ echo '</div>';
+ }
+ if (isset($row['statistics']['budget'])) {
+ echo '<div><i class="icon-shopping-cart" style="float:left"></i><p style="margin-left:16px;">';
+ $keys = array_keys($row['statistics']['budget']);
+ $lastkey = $keys[count($keys) - 1];
+ echo "$" . number_format(floatval($row['statistics']['budget'][$lastkey]['value'])) . ' <small>(' . $lastkey . ' budget)</small>';
+ echo '</div>';
+ }
echo ' </ul>
</div><!--/.well -->
</div><!--/span-->
<div class="span9">';
- echo '<div itemscope itemtype="http://schema.org/GovernmentOrganization" typeof="schema:GovernmentOrganization" about="#' . $row['_id'] . '">';
- echo '<div class="hero-unit">
+ echo '<div itemscope itemtype="http://schema.org/GovernmentOrganization" typeof="schema:GovernmentOrganization" about="#' . $row['_id'] . '">';
+ echo '<div class="hero-unit">
<h1 itemprop="name">' . $row['name'] . '</h1>';
- if (isset($row['description'])) {
- echo '<p>'.$row['description'].'</p>';
- }
- echo '</div><table width="100%">';
- echo "<tr><th>Field Name</th><th>Field Value</th></tr>";
- }
- if ($mode == "edit") {
- ?>
- <input id="addfield" type="button" value="Add Field"/>
- <script>
- window.onload = function() {
- $(document).ready(function() {
- // put all your jQuery goodness in here.
- // http://charlie.griefer.com/blog/2009/09/17/jquery-dynamically-adding-form-elements/
- $('#addfield').click(function() {
- var field_name=window.prompt("fieldname?","");
- if (field_name !="") {
- $('#submitbutton').before($('<span></span>')
- .append("<label>"+field_name+"</label>")
- .append("<input class='input-text' type='text' id='"+field_name+"' name='"+field_name+"'/>")
- );
- }
- });
- });
- };
- </script>
- <form id="editform" class="nice" method="post">
- <?php
+ if (isset($row['description'])) {
+ echo '<p>' . $row['description'] . '</p>';
+ }
+ echo '</div><table width="100%">';
+ echo "<tr><th>Field Name</th><th>Field Value</th></tr>";
+}
+if ($mode == "edit") {
+?>
+<input id="addfield" type="button" value="Add Field"/>
+<script>
+ window.onload = function () {
+ $(document).ready(function () {
+ // put all your jQuery goodness in here.
+ // http://charlie.griefer.com/blog/2009/09/17/jquery-dynamically-adding-form-elements/
+ $('#addfield').click(function () {
+ var field_name = window.prompt("fieldname?", "");
+ if (field_name != "") {
+ $('#submitbutton').before($('<span></span>')
+ .append("<label>" + field_name + "</label>")
+ .append("<input class='input-text' type='text' id='" + field_name + "' name='" + field_name + "'/>")
+ );
+ }
+ });
+ });
+ };
+</script>
+<form id="editform" class="nice" method="post">
+ <?php
}
foreach ($row as $key => $value) {
@@ -223,7 +226,7 @@
}
if ($mode == "view") {
echo "</table></div>";
- echo ' </div><!--/span-->
+ echo ' </div><!--/span-->
</div><!--/row-->
</div><!--/span-->
</div><!--/row-->';
@@ -231,16 +234,16 @@
if ($mode == "edit") {
echo '<input id="submitbutton" type="submit"/></form>';
}
-} else {
+ } else {
// show all list
- include_header('Agencies');
- echo ' <div class="container-fluid">
+ include_header('Agencies');
+ echo ' <div class="container-fluid">
<div class="row-fluid">
<div class="span3">
<div class="well sidebar-nav">
<ul class="nav nav-list">
<li class="nav-header">Sidebar</li>';
- echo ' </ul>
+ echo ' </ul>
</div><!--/.well -->
</div><!--/span-->
<div class="span9">
@@ -251,28 +254,28 @@
</div>
<div class="row-fluid">
<div class="span4">';
- try {
- $rows = $db->get_view("app", "byCanonicalName")->rows;
- //print_r($rows);
- $rowCount = count($rows);
- foreach ($rows as $i => $row) {
- if ($i % ($rowCount/3) == 0 && $i != 0 && $i != $rowCount -2 ) echo '</div><div class="span4">';
- // print_r($row);
- echo '<span itemscope itemtype="http://schema.org/GovernmentOrganization" typeof="schema:GovernmentOrganization foaf:Organization" about="getAgency.php?id=' . $row->value->_id . '">
+ try {
+ $rows = $db->get_view("app", "byCanonicalName")->rows;
+ //print_r($rows);
+ $rowCount = count($rows);
+ foreach ($rows as $i => $row) {
+ if ($i % ($rowCount / 3) == 0 && $i != 0 && $i != $rowCount - 2) echo '</div><div class="span4">';
+ // print_r($row);
+ echo '<span itemscope itemtype="http://schema.org/GovernmentOrganization" typeof="schema:GovernmentOrganization foaf:Organization" about="getAgency.php?id=' . $row->value->_id . '">
<a href="getAgency.php?id=' . $row->value->_id . '" rel="schema:url foaf:page" property="schema:name foaf:name" itemprop="url"><span itemprop="name">' .
- (isset($row->value->name) ? $row->value->name : "ERROR NAME MISSING")
- . '</span></a></span><br><br>';
- }
-
- } catch (SetteeRestClientException $e) {
- setteErrorHandler($e);
- }
- echo ' </div><!--/span-->
+ (isset($row->value->name) ? $row->value->name : "ERROR NAME MISSING")
+ . '</span></a></span><br><br>';
+ }
+
+ } catch (SetteeRestClientException $e) {
+ setteErrorHandler($e);
+ }
+ echo ' </div><!--/span-->
</div><!--/row-->
</div><!--/span-->
</div><!--/row-->';
-}
-
-include_footer();
-?>
-
+ }
+
+ include_footer();
+ ?>
+
--- a/include/common.inc.php
+++ b/include/common.inc.php
@@ -19,7 +19,7 @@
Requests::register_autoloader();
$ENV = "DEV";
-if (isset($_SERVER['SERVER_NAME']) && $_SERVER['SERVER_NAME'] != 'localhost') {
+if (false && isset($_SERVER['SERVER_NAME']) && $_SERVER['SERVER_NAME'] != 'localhost') {
require $basePath."lib/amon-php/amon.php";
Amon::config(array('address'=> 'http://127.0.0.1:2464',
--- a/include/couchdb.inc.php
+++ b/include/couchdb.inc.php
@@ -14,8 +14,8 @@
} else
if (php_uname('n') == "ikurt-20") {
- $serverAddr = 'http://192.168.1.113:5984/';
- //$serverAddr = 'http://127.0.0.1:5984/';
+ //$serverAddr = 'http://192.168.1.113:5984/';
+ $serverAddr = 'http://127.0.0.1:5984/';
} else {
$serverAddr = 'http://127.0.0.1:5984/';
}
--- a/include/template.inc.php
+++ b/include/template.inc.php
@@ -28,6 +28,9 @@
.sidebar-nav {
padding: 9px 0;
}
+ .flotr-dummy-div {
+ margin-left: -999px;
+ }
</style>
<link href="<?php echo $basePath ?>css/bootstrap-responsive.min.css" rel="stylesheet">
<!--[if lt IE 9]>
@@ -54,6 +57,7 @@
<div class="nav-collapse collapse">
<ul class="nav">
<li><a href="getAgency.php">Agencies</a></li>
+ <li><a href="ranking.php">Open Gov Ranking</a></li>
<li><a href="headcount.php">Employee Headcount Graph</a></li>
<li><a href="budget.php">Budget Graph</a></li>
<li><a href="about.php">About/FAQ</a></li>
--- a/js/bubbletree
+++ b/js/bubbletree
--- a/js/flotr2
+++ b/js/flotr2
--- a/js/sigma
+++ b/js/sigma
--- a/lib/phpquery
+++ b/lib/phpquery
--- /dev/null
+++ b/ranking.php
@@ -1,1 +1,213 @@
-
+<?php
+include_once('include/common.inc.php');
+include_header('Open Gov Rankings');
+$db = $server->get_db('disclosr-agencies');
+?>
+<div class="foundation-header">
+ <h1><a href="about.php">Open Government Rankings</a></h1>
+ <h4 class="subheader"></h4>
+</div>
+<table>
+ <?php
+ $agenciesdb = $server->get_db('disclosr-agencies');
+ //$docsdb = $server->get_db('disclosr-documents');
+ $scoredagencies = Array();
+ $scores = Array();
+ $columnKeys = Array();
+
+ try {
+ $rows = $agenciesdb->get_view("app", "all", null, true)->rows;
+
+
+ if ($rows) {
+ foreach ($rows as $row) {
+ $columns = Array();
+ foreach ($row->value as $key => $value) {
+ if ((strstr($key, "has") || strstr($key, "URL")) && $key != "rtkURLs") {
+ //echo "$key<br>";
+ $columns[$key] = $value;
+ }
+ }
+ //print_r(array_keys($columns));
+ $columnKeys = array_unique(array_merge($columnKeys, array_keys($columns)));
+ //print_r($columnKeys);
+ $score = count($columns);
+ $scores[$score]++;
+ $scoredagencies[] = Array("id"=> $row->key, "website"=> $row->value->website, "name" => $row->value->name, "columns" => $columns, "score" => $score);
+ }
+ }
+
+ } catch (SetteeRestClientException $e) {
+ setteErrorHandler($e);
+ }
+ function cmp($a, $b)
+ {
+ if ($a['score'] == $b['score']) {
+ return strcmp($a['name'], $b['name']);
+ }
+ return ($a['score'] > $b['score']) ? -1 : 1;
+ }
+
+ usort($scoredagencies, "cmp");
+ echo "<tr>";
+ echo "<th>Agency Name</th>";
+ echo "<th>Score</th>";
+ foreach ($columnKeys as $columnID) {
+ echo "<th>" . (isset($schemas['agency']["properties"][$columnID]['x-title']) ? $schemas['agency']["properties"][$columnID]['x-title'] : "<i>$columnID</i>") . "</th>";
+ }
+ echo "</tr>";
+ foreach ($scoredagencies as $scoredagency) {
+ echo "<tr>";
+ echo "<td><b><a href='getAgency.php?id=" . $scoredagency['id'] . "'>". $scoredagency['name'] . "</a></b></td>";
+ echo "<td><b>" . $scoredagency['score'] . "</b></td>";
+ foreach ($columnKeys as $key) {
+ echo "<td style='text-align: center;'>";
+ if (isset($scoredagency['columns'][$key])) {
+ $value = $scoredagency['columns'][$key];
+ if (is_array($value)) {
+ if (count($value) == 1) {
+ $href = $value[0];
+ } else {
+ $href = $value[0];
+ }
+
+ } else {
+ $href = $value;
+ }
+ if ($href[0] == "@") {
+ $href = str_replace("@","https://twitter.com/",$href);
+ }
+ //$href= urlencode($href);
+
+ echo "<font color='lightgreen'>";
+
+ if (strstr($href, "http")) {
+ echo "<a title='Yes' href='$href' style='color:lightgreen;'>✓</a>";
+ } else {
+ echo "✓";
+ }
+
+ echo "</font>";
+ } else {
+ echo "<font color='orange'><abbr title='No'>✘</abbr></font>";
+ }
+ echo "</td>";
+ }
+ echo "</tr>\n";
+ }
+ ?>
+</table><br>
+<div id="criteria" style="width:500px;height:900px;"></div>
+<div id="scores" style="width:900px;height:500px;"></div>
+<script id="source">
+ window.onload = function () {
+ $(document).ready(function () {
+ var d1 = [];
+ var scorelabels = [];
+ <?php
+ try {
+ $rows = $db->get_view("app", "scoreHas?group=true", null, true)->rows;
+
+
+ $dataValues = Array();
+ foreach ($rows as $row) {
+ $dataValues[$row->value] = $row->key;
+ }
+ $i = 0;
+ ksort($dataValues);
+ foreach ($dataValues as $value => $key) {
+
+ echo " d1.push([$value, $i]);" . PHP_EOL;
+ echo " scorelabels.push('$key');" . PHP_EOL;
+ $i++;
+ }
+ } catch (SetteeRestClientException $e) {
+ setteErrorHandler($e);
+ }
+ ?>
+ function scoretrackformatter(obj) {
+ if (scorelabels[Math.floor(obj.y)]) {
+ return (scorelabels[Math.floor(obj.y)]) + "=" + obj.x;
+
+ } else {
+ return "";
+ }
+ }
+
+ function scoretickformatter(val, axis) {
+ if (scorelabels[Math.floor(val)]) {
+ return (scorelabels[Math.floor(val)]) ;
+
+ } else {
+ return "";
+ }
+ }
+
+ Flotr.draw(document.getElementById("criteria"), [
+ {data: d1}
+ ], {
+ title: 'Total count of agencies with criteria',
+ HtmlText: true,
+ bars: {
+ show: true,
+ horizontal: true
+ },
+ mouse: {
+ track: true,
+ relative: true,
+ trackFormatter: scoretrackformatter
+ }, yaxis: {
+ autoscaling: true,
+ minorTickFreq: 0.6,
+ noTicks: scorelabels.length,
+ tickFormatter: scoretickformatter
+ },
+ xaxis: {
+ autoscaling: true
+
+ }
+ });
+
+ var d2 = [];
+ <?php
+ try {
+
+ ksort($scores);
+ foreach ($scores as $key => $value) {
+
+ echo " d2.push([$key,$value]);" . PHP_EOL;
+ $i++;
+ }
+ } catch (SetteeRestClientException $e) {
+ setteErrorHandler($e);
+ }
+ ?>
+
+
+ Flotr.draw(document.getElementById("scores"), [
+ {data: d2}
+ ], {
+ title: 'Frequency distribution of Scores',
+ HtmlText: true,
+ bars: {
+ show: true
+ },
+ mouse: {
+ track: true,
+ relative: true
+ }, yaxis: {
+ autoscaling: true
+ },
+ xaxis: {
+ autoscaling: true
+
+ }
+ });
+
+ });
+ };
+</script>
+<?php
+include_footer();
+?>
+