gitphp 0.2.9.1 :: disclosr.git/commitdiff

fix wcag validation tester

Former-commit-id: 7194cc6a6482caa4c20ac597289ff96b748223c3

3 files changed: (show all)
admin/refreshDesignDoc.php
admin/validation.py
documents/scrape.py

file:a/admin/refreshDesignDoc.php -> file:b/admin/refreshDesignDoc.php

file:a/admin/validation.py -> file:b/admin/validation.py

#http://packages.python.org/CouchDB/client.html	#http://packages.python.org/CouchDB/client.html
import couchdb	import couchdb
import json	import json
import pprint	import pprint
import re	import re
from tidylib import tidy_document	from tidylib import tidy_document

couch = couchdb.Server('http://127.0.0.1:5984/')	couch = couchdb.Server('http://192.168.1.113:5984/')

# select database	# select database
docsdb = couch['disclosr-documents']	docsdb = couch['disclosr-documents']

def f(x):	def f(x):
invalid = re.compile(r"ensure\|testing\|flicker\|updating\|longdesc\|Accessibility Checks\|not recognized")	invalid = re.compile(r"ensure\|testing\|flicker\|updating\|longdesc\|Accessibility Checks\|not recognized\|noscript\|audio")
valid = re.compile(r"line")	valid = re.compile(r"line")
return (not invalid.search(x)) and valid.search(x) and x != ''	return (not invalid.search(x)) and valid.search(x) and x != ''

for row in docsdb.view('app/getValidationRequired'):	for row in docsdb.view('app/getValidationRequired'):
print row.id	print row.id
html = docsdb.get_attachment(row.id,row.value.iterkeys().next()).read()	html = docsdb.get_attachment(row.id,row.value.iterkeys().next()).read()
#print html	#print html
document, errors = tidy_document(html,options={'accessibility-check':1,'show-warnings':0,'markup':0},keep_doc=True)	document, errors = tidy_document(html,options={'accessibility-check':1,'show-warnings':0,'markup':0},keep_doc=True)
#http://www.aprompt.ca/Tidy/accessibilitychecks.html	#http://www.aprompt.ca/Tidy/accessibilitychecks.html
#print document	#print document
errors = '\n'.join(filter(f,errors.split('\n')))	errors = '\n'.join(filter(f,errors.split('\n')))
#print errors	#print errors
doc = docsdb.get(row.id)	doc = docsdb.get(row.id)
doc['validation'] = errors	doc['validation'] = errors
docsdb.save(doc)	docsdb.save(doc)

file:a/documents/scrape.py -> file:b/documents/scrape.py

#http://packages.python.org/CouchDB/client.html	#http://packages.python.org/CouchDB/client.html
import couchdb	import couchdb
import urllib2	import urllib2
from BeautifulSoup import BeautifulSoup	from BeautifulSoup import BeautifulSoup
import re	import re
import hashlib	import hashlib
from urlparse import urljoin	from urlparse import urljoin
import time	import time
import os	import os
import mimetypes	import mimetypes
import urllib	import urllib
import urlparse	import urlparse
	import socket


def mkhash(input):	def mkhash(input):
return hashlib.md5(input).hexdigest().encode("utf-8")	return hashlib.md5(input).hexdigest().encode("utf-8")

def canonurl(url):	def canonurl(url):
r"""Return the canonical, ASCII-encoded form of a UTF-8 encoded URL, or ''	r"""Return the canonical, ASCII-encoded form of a UTF-8 encoded URL, or ''
if the URL looks invalid.	if the URL looks invalid.
>>> canonurl('\xe2\x9e\xa1.ws') # tinyarro.ws	>>> canonurl('\xe2\x9e\xa1.ws') # tinyarro.ws
'http://xn--hgi.ws/'	'http://xn--hgi.ws/'
"""	"""
# strip spaces at the ends and ensure it's prefixed with 'scheme://'	# strip spaces at the ends and ensure it's prefixed with 'scheme://'
url = url.strip()	url = url.strip()
if not url:	if not url:
return ''	return ''
if not urlparse.urlsplit(url).scheme:	if not urlparse.urlsplit(url).scheme:
url = 'http://' + url	url = 'http://' + url

# turn it into Unicode	# turn it into Unicode
#try:	#try:
# url = unicode(url, 'utf-8')	# url = unicode(url, 'utf-8')
#except UnicodeDecodeError:	#except UnicodeDecodeError:
# return '' # bad UTF-8 chars in URL	# return '' # bad UTF-8 chars in URL

# parse the URL into its components	# parse the URL into its components
parsed = urlparse.urlsplit(url)	parsed = urlparse.urlsplit(url)
scheme, netloc, path, query, fragment = parsed	scheme, netloc, path, query, fragment = parsed

# ensure scheme is a letter followed by letters, digits, and '+-.' chars	# ensure scheme is a letter followed by letters, digits, and '+-.' chars
if not re.match(r'[a-z][-+.a-z0-9]*$', scheme, flags=re.I):	if not re.match(r'[a-z][-+.a-z0-9]*$', scheme, flags=re.I):
return ''	return ''
scheme = str(scheme)	scheme = str(scheme)

# ensure domain and port are valid, eg: sub.domain.<1-to-6-TLD-chars>[:port]	# ensure domain and port are valid, eg: sub.domain.<1-to-6-TLD-chars>[:port]
match = re.match(r'(.+\.[a-z0-9]{1,6})(:\d{1,5})?$', netloc, flags=re.I)	match = re.match(r'(.+\.[a-z0-9]{1,6})(:\d{1,5})?$', netloc, flags=re.I)
if not match:	if not match:
return ''	return ''
domain, port = match.groups()	domain, port = match.groups()
netloc = domain + (port if port else '')	netloc = domain + (port if port else '')
netloc = netloc.encode('idna')	netloc = netloc.encode('idna')

# ensure path is valid and convert Unicode chars to %-encoded	# ensure path is valid and convert Unicode chars to %-encoded
if not path:	if not path:
path = '/' # eg: 'http://google.com' -> 'http://google.com/'	path = '/' # eg: 'http://google.com' -> 'http://google.com/'
path = urllib.quote(urllib.unquote(path.encode('utf-8')), safe='/;')	path = urllib.quote(urllib.unquote(path.encode('utf-8')), safe='/;')

# ensure query is valid	# ensure query is valid
query = urllib.quote(urllib.unquote(query.encode('utf-8')), safe='=&?/')	query = urllib.quote(urllib.unquote(query.encode('utf-8')), safe='=&?/')

# ensure fragment is valid	# ensure fragment is valid
fragment = urllib.quote(urllib.unquote(fragment.encode('utf-8')))	fragment = urllib.quote(urllib.unquote(fragment.encode('utf-8')))

# piece it all back together, truncating it to a maximum of 4KB	# piece it all back together, truncating it to a maximum of 4KB
url = urlparse.urlunsplit((scheme, netloc, path, query, fragment))	url = urlparse.urlunsplit((scheme, netloc, path, query, fragment))
return url[:4096]	return url[:4096]

def fullurl(url,href):	def fullurl(url,href):
href = href.replace(" ","%20")	href = href.replace(" ","%20")
href = re.sub('#.*$','',href)	href = re.sub('#.*$','',href)
return urljoin(url,href)	return urljoin(url,href)

#http://diveintopython.org/http_web_services/etags.html	#http://diveintopython.org/http_web_services/etags.html
class NotModifiedHandler(urllib2.BaseHandler):	class NotModifiedHandler(urllib2.BaseHandler):
def http_error_304(self, req, fp, code, message, headers):	def http_error_304(self, req, fp, code, message, headers):
addinfourl = urllib2.addinfourl(fp, headers, req.get_full_url())	addinfourl = urllib2.addinfourl(fp, headers, req.get_full_url())
addinfourl.code = code	addinfourl.code = code
return addinfourl	return addinfourl

def getLastAttachment(docsdb,url):	def getLastAttachment(docsdb,url):
hash = mkhash(url)	hash = mkhash(url)
doc = docsdb.get(hash)	doc = docsdb.get(hash)
if doc != None:	if doc != None:
last_attachment_fname = doc["_attachments"].keys()[-1]	last_attachment_fname = doc["_attachments"].keys()[-1]
last_attachment = docsdb.get_attachment(doc,last_attachment_fname)	last_attachment = docsdb.get_attachment(doc,last_attachment_fname)
return last_attachment	return last_attachment
else:	else:
return None	return None

def fetchURL(docsdb, url, fieldName, agencyID, scrape_again=True):	def fetchURL(docsdb, url, fieldName, agencyID, scrape_again=True):
url = canonurl(url)	url = canonurl(url)
hash = mkhash(url)	hash = mkhash(url)
req = urllib2.Request(url)	req = urllib2.Request(url)
print "Fetching %s (%s)" % (url,hash)	print "Fetching %s (%s)" % (url,hash)
if url.startswith("mailto") or url.startswith("javascript") or url.startswith("#") or url == None or url == "":	if url.startswith("mailto") or url.startswith("javascript") or url.startswith("#") or url == None or url == "":
print "Not a valid HTTP url"	print "Not a valid HTTP url"
return (None,None,None)	return (None,None,None)
doc = docsdb.get(hash)	doc = docsdb.get(hash)
if doc == None:	if doc == None:
doc = {'_id': hash, 'agencyID': agencyID, 'url': url, 'fieldName':fieldName}	doc = {'_id': hash, 'agencyID': agencyID, 'url': url, 'fieldName':fieldName, 'type': 'website'}
else:	else:
if (('page_scraped' in doc) and (time.time() - doc['page_scraped']) < 602414*1000):	if (('page_scraped' in doc) and (time.time() - doc['page_scraped']) < 602414*1000):
print "Uh oh, trying to scrape URL again too soon!"+hash	print "Uh oh, trying to scrape URL again too soon!"+hash
last_attachment_fname = doc["_attachments"].keys()[-1]	last_attachment_fname = doc["_attachments"].keys()[-1]
last_attachment = docsdb.get_attachment(doc,last_attachment_fname)	last_attachment = docsdb.get_attachment(doc,last_attachment_fname)
content = last_attachment	content = last_attachment
return (doc['url'],doc['mime_type'],content.read())	return (doc['url'],doc['mime_type'],content.read())
if scrape_again == False:	if scrape_again == False:
print "Not scraping this URL again as requested"	print "Not scraping this URL again as requested"
return (doc['url'],doc['mime_type'],content.read())	return (doc['url'],doc['mime_type'],content.read())

req.add_header("User-Agent", "Mozilla/4.0 (compatible; Prometheus webspider; owner maxious@lambdacomplex.org)")	req.add_header("User

<?php	<?php

require_once '../include/common.inc.php';	require_once '../include/common.inc.php';
//function createFOIDocumentsDesignDoc() {	//function createFOIDocumentsDesignDoc() {

$foidb = $server->get_db('disclosr-foidocuments');	$foidb = $server->get_db('disclosr-foidocuments');
$obj = new stdClass();	$obj = new stdClass();
$obj->_id = "_design/" . urlencode("app");	$obj->_id = "_design/" . urlencode("app");
$obj->language = "javascript";	$obj->language = "javascript";
$obj->views->all->map = "function(doc) { emit(doc._id, doc); };";	$obj->views->all->map = "function(doc) { emit(doc._id, doc); };";
$obj->views->byDate->map = "function(doc) { emit(doc.date, doc); };";	$obj->views->byDate->map = "function(doc) { emit(doc.date, doc); };";
$obj->views->byDateMonthYear->map = "function(doc) { emit(doc.date, doc); };";	$obj->views->byDateMonthYear->map = "function(doc) { emit(doc.date, doc); };";
$obj->views->byDateMonthYear->reduce = "_count";	$obj->views->byDateMonthYear->reduce = "_count";
$obj->views->byAgencyID->map = "function(doc) { emit(doc.agencyID, doc); };";	$obj->views->byAgencyID->map = "function(doc) { emit(doc.agencyID, doc); };";
$obj->views->byAgencyID->reduce = "_count";	$obj->views->byAgencyID->reduce = "_count";
$obj->views->fieldNames->map = '	$obj->views->fieldNames->map = '
function(doc) {	function(doc) {
for(var propName in doc) {	for(var propName in doc) {
emit(propName, doc._id);	emit(propName, doc._id);
}	}

}';	}';
$obj->views->fieldNames->reduce = 'function (key, values, rereduce) {	$obj->views->fieldNames->reduce = 'function (key, values, rereduce) {
return values.length;	return values.length;
}';	}';
// allow safe updates (even if slightly slower due to extra: rev-detection check).	// allow safe updates (even if slightly slower due to extra: rev-detection check).
$foidb->save($obj, true);	$foidb->save($obj, true);


//function createDocumentsDesignDoc() {	//function createDocumentsDesignDoc() {
$docdb = $server->get_db('disclosr-documents');	$docdb = $server->get_db('disclosr-documents');

$obj = new stdClass();	$obj = new stdClass();
$obj->_id = "_design/" . urlencode("app");	$obj->_id = "_design/" . urlencode("app");
$obj->language = "javascript";	$obj->language = "javascript";
$obj->views->web_server->map = "function(doc) {\n emit(doc.web_server, 1);\n}";	$obj->views->web_server->map = "function(doc) {\n emit(doc.web_server, 1);\n}";
$obj->views->web_server->reduce = "_sum";	$obj->views->web_server->reduce = "_sum";
$obj->views->byAgency->map = "function(doc) {\n emit(doc.agencyID, 1);\n}";	$obj->views->byAgency->map = "function(doc) {\n emit(doc.agencyID, 1);\n}";
$obj->views->byAgency->reduce = "_sum";	$obj->views->byAgency->reduce = "_sum";
$obj->views->byURL->map = "function(doc) {\n emit(doc.url, doc);\n}";	$obj->views->byURL->map = "function(doc) {\n emit(doc.url, doc);\n}";
$obj->views->agency->map = "function(doc) {\n emit(doc.agencyID, doc);\n}";	$obj->views->agency->map = "function(doc) {\n emit(doc.agencyID, doc);\n}";
$obj->views->byWebServer->map = "function(doc) {\n emit(doc.web_server, doc);\n}";	$obj->views->byWebServer->map = "function(doc) {\n emit(doc.web_server, doc);\n}";
$obj->views->getValidationRequired = "function(doc) {\nif (doc.mime_type == \"text/html\" \n&& typeof(doc.validation) == \"undefined\") {\n emit(doc._id, doc._attachments);\n}\n}";	$obj->views->getValidationRequired->map = "function(doc) {\nif (doc.mime_type == \"text/html\" \n&& typeof(doc.validation) == \"undefined\") {\n emit(doc._id, doc._attachments);\n}\n}";
	$docdb->save($obj, true);




//function createAgencyDesignDoc() {	//function createAgencyDesignDoc() {
$db = $server->get_db('disclosr-agencies');	$db = $server->get_db('disclosr-agencies');
$obj = new stdClass();	$obj = new stdClass();
$obj->_id = "_design/" . urlencode("app");	$obj->_id = "_design/" . urlencode("app");
$obj->language = "javascript";	$obj->language = "javascript";
$obj->views->all->map = "function(doc) { emit(doc._id, doc); };";	$obj->views->all->map = "function(doc) { emit(doc._id, doc); };";
$obj->views->byABN->map = "function(doc) { emit(doc.abn, doc); };";	$obj->views->byABN->map = "function(doc) { emit(doc.abn, doc); };";
$obj->views->byCanonicalName->map = "function(doc) {	$obj->views->byCanonicalName->map = "function(doc) {
if (doc.parentOrg \|\| doc.orgType == 'FMA-DepartmentOfState') {	if (doc.parentOrg \|\| doc.orgType == 'FMA-DepartmentOfState') {
emit(doc.name, doc);	emit(doc.name, doc);
}	}
};";	};";
$obj->views->byDeptStateName->map = "function(doc) {	$obj->views->byDeptStateName->map = "function(doc) {
if (doc.orgType == 'FMA-DepartmentOfState') {	if (doc.orgType == 'FMA-DepartmentOfState') {
emit(doc.name, doc._id);	emit(doc.name, doc._id);
}	}
};";	};";
$obj->views->parentOrgs->map = "function(doc) {	$obj->views->parentOrgs->map = "function(doc) {
if (doc.parentOrg) {	if (doc.parentOrg) {
emit(doc._id, doc.parentOrg);	emit(doc._id, doc.parentOrg);
}	}
};";	};";
$obj->views->byName->map = 'function(doc) {	$obj->views->byName->map = 'function(doc) {
if (typeof(doc["status"]) == "undefined" \|\| doc["status"] != "suspended") {	if (typeof(doc["status"]) == "undefined" \|\| doc["status"] != "suspended") {
emit(doc.name, doc._id);	emit(doc.name, doc._id);
if (typeof(doc.shortName) != "undefined" && doc.shortName != doc.name) {	if (typeof(doc.shortName) != "undefined" && doc.shortName != doc.name) {
emit(doc.shortName, doc._id);	emit(doc.shortName, doc._id);
}	}
for (name in doc.otherNames) {	for (name in doc.otherNames) {
if (doc.otherNames[name] != "" && doc.otherNames[name] != doc.name) {	if (doc.otherNames[name] != "" && doc.otherNames[name] != doc.name) {
emit(doc.otherNames[name], doc._id);	emit(doc.otherNames[name], doc._id);
}	}
}	}
for (name in doc.foiBodies) {	for (name in doc.foiBodies) {
if (doc.foiBodies[name] != "" && doc.foiBodies[name] != doc.name) {	if (doc.foiBodies[name] != "" && doc.foiBodies[name] != doc.name) {
emit(doc.foiBodies[name], doc._id);	emit(doc.foiBodies[name], doc._id);
}	}
}	}
for (name in doc.positions) {	for (name in doc.positions) {
if (doc.positions[name] != "" && doc.positions[name] != doc.name) {	if (doc.positions[name] != "" && doc.positions[name] != doc.name) {
emit(doc.positions[name], doc._id);	emit(doc.positions[name], doc._id);
}	}
}	}
}	}
};';	};';

$obj->views->foiEmails->map = "function(doc) {	$obj->views->foiEmails->map = "function(doc) {
emit(doc._id, doc.foiEmail);	emit(doc._id, doc.foiEmail);
};";	};";

$obj->views->byLastModified->map = "function(doc) { emit(doc.metadata.lastModified, doc); }";	$obj->views->byLastModified->map = "function(doc) { emit(doc.metadata.lastModified, doc); }";
$obj->views->getActive->map = 'function(doc) { if (doc.status == "active") { emit(doc._id, doc); } };';	$obj->views->getActive->map = 'function(doc) { if (doc.status == "active") { emit(doc._id, doc); } };';
$obj->views->getSuspended->map = 'function(doc) { if (doc.status == "suspended") { emit(doc._id, doc); } };';	$obj->views->getSuspended->map = 'function(doc) { if (doc.status == "suspended") { emit(doc._id, doc); } };';
$obj->views->getScrapeRequired->map = "function(doc) {	$obj->views->getScrapeRequired->map = "function(doc) {

var lastScrape = Date.parse(doc.metadata.lastScraped);	var lastScrape = Date.parse(doc.metadata.lastScraped);

var today = new Date();	var today = new Date();

if (!lastScrape \|\| lastScrape.getTime() + 1000 != today.getTime()) {	if (!lastScrape \|\| lastScrape.getTime() + 1000 != today.getTime()) {
emit(doc._id, doc);	emit(doc._id, doc);
}	}

};";	};";
$obj->views->showNamesABNs->map = "function(doc) { emit(doc._id, {name: doc.name, abn: doc.abn}); };";	$obj->views->showNamesABNs->map = "function(doc) { emit(doc._id, {name: doc.name, abn: doc.abn}); };";
$obj->views->getConflicts->map = "function(doc) {	$obj->views->getConflicts->map = "function(doc) {
if (doc._conflicts) {	if (doc._conflicts) {
emit(null, [doc._rev].concat(doc._conflicts));	emit(null, [doc._rev].concat(doc._conflicts));
}	}
}";	}";
$obj->views->getStatistics->map =	$obj->views->getStatistics->map =
"function(doc) {	"function(doc) {
if (doc.statistics) {	if (doc.statistics) {
for (var statisticSet in doc.statistics) {	for (var statisticSet in doc.statistics) {
for (var statisticPeriod in doc.statistics[statisticSet]) {	for (var statisticPeriod in doc.statistics[statisticSet]) {
emit([statisticSet,statisticPeriod], doc.statistics[statisticSet][statisticPeriod]['value']);	emit([statisticSet,statisticPeriod], doc.statistics[statisticSet][statisticPeriod]['value']);
}	}
}	}
}	}
}";	}";
$obj->views->getStatistics->reduce = '_sum';	$obj->views->getStatistics->reduce = '_sum';
// http://stackoverflow.com/questions/646628/javascript-startswith	// http://stackoverflow.com/questions/646628/javascript-startswith
$obj->views->score->map = 'if(!String.prototype.startsWith){	$obj->views->score->map = 'if(!String.prototype.startsWith){
String.prototype.startsWith = function (str) {	String.prototype.startsWith = function (str) {
return !this.indexOf(str);	return !this.indexOf(str);
}	}
}	}

function(doc) {	function(doc) {
count = 0;	count = 0;
if (doc["status"] != "suspended") {	if (doc["status"] != "suspended") {
for(var propName in doc) {	for(var propName in doc) {
if(typeof(doc[propName]) != "undefined" && doc[propName] != "") {	if(typeof(doc[propName]) != "undefined" && doc[propName] != "") {
count++;	count++;
}	}
}	}
portfolio = doc.parentOrg;	portfolio = doc.parentOrg;
if (doc.orgType == "FMA-DepartmentOfState") {	if (doc.orgType == "FMA-DepartmentOfState") {
portfolio = doc._id;	portfolio = doc._id;
}	}
if (doc.orgType == "Court-Commonwealth" \|\| doc.orgType == "FMA-DepartmentOfParliament") {	if (doc.orgType == "Court-Commonwealth" \|\| doc.orgType == "FMA-DepartmentOfParliament") {
portfolio = doc.orgType;	portfolio = doc.orgType;
}	}
emit(count+doc._id, {id:doc._id, name: doc.name, score:count, orgType: doc.orgType, portfolio:portfolio});	emit(count+doc._id, {id:doc._id, name: doc.name, score:count, orgType: doc.orgType, portfolio:portfolio});
}	}
}';	}';
$obj->views->scoreHas->map = 'if(!String.prototype.startsWith){	$obj->views->scoreHas->map = 'if(!String.prototype.startsWith){
String.prototype.startsWith = function (str) {	String.prototype.startsWith = function (str) {
return !this.indexOf(str);	return !this.indexOf(str);
}	}
}	}
if(!String.prototype.endsWith){	if(!String.prototype.endsWith){
String.prototype.endsWith = function(suffix) {	String.prototype.endsWith = function(suffix) {
return this.indexOf(suffix, this.length - suffix.length) !== -1;	return this.indexOf(suffix, this.length - suffix.length) !== -1;
};	};
}	}
function(doc) {	function(doc) {
if (typeof(doc["status"]) == "undefined" \|\| doc["status"] != "suspended") {	if (typeof(doc["status"]) == "undefined" \|\| doc["status"] != "suspended") {
for(var propName in doc) {	for(var propName in doc) {
if(typeof(doc[propName]) != "undefined" && (propName.startsWith("has") \|\| propName.endsWith("URL"))) {	if(typeof(doc[propName]) != "undefined" && (propName.startsWith("has") \|\| propName.endsWith("URL"))) {
emit(propName, 1);	emit(propName, 1);
}	}
}	}
emit("total", 1);	emit("total", 1);
}	}
}';	}';
$obj->views->scoreHas->reduce = '_sum';	$obj->views->scoreHas->reduce = '_sum';
$obj->views->fieldNames->map = '	$obj->views->fieldNames->map = '
function(doc) {	function(doc) {
for(var propName in doc) {	for(var propName in doc) {
emit(propName, doc._id);	emit(propName, doc._id);
}	}

}';	}';
$obj->views->fieldNames->reduce = '_count';	$obj->views->fieldNames->reduce = '_count';
// allow safe updates (even if slightly slower due to extra: rev-detection check).	// allow safe updates (even if slightly slower due to extra: rev-detection check).
$db->save($obj, true);	$db->save($obj, true);
?>	?>