From: Alex Sadleir Date: Thu, 27 Mar 2014 12:52:48 +0000 Subject: beginnings of NAA data import X-Git-Url: https://maxious.lambdacomplex.org/git/?p=disclosr.git&a=commitdiff&h=9c984992bb7c34c20eb5ab66d9399961832f22b7 --- beginnings of NAA data import Former-commit-id: 111602028550f26c5952c2705444c00119d4d440 --- --- a/admin/importOAICFOIrequests.php +++ b/admin/importOAICFOIrequests.php @@ -17,13 +17,13 @@ if ($row >= 1) { // print_r($data); $name = trim($data[2]); -// echo "$name
"; + echo "$name
"; if ($data[0] != "TOTALS" && $data[0] != "") { if (isset($nametoid[$name])) { $id = $nametoid[$name]; $timePeriod = $data[0] . "-Q" . $data[1]; -// echo "$timePeriod
"; + echo "$timePeriod
"; unset($data[0]); unset($data[1]); unset($data[2]); @@ -38,13 +38,10 @@ $result = Array("source" => "http://data.gov.au/dataset/freedom-of-information-quarterly-request-and-review-statistical-data-2011-12/"); foreach ($data as $key => $datum) { if ($datum != 0) { -// tODO prefix header with "FOI" -if (isset($stats[$id][$timePeriod][$key])) $datum += $stats[$id][$timePeriod][$key]; $result[trim($headers[$key])] = $datum; } } $stats[$id][$timePeriod] = $result; -// TODO merge if already exists //print_r($stats); } else { echo "
ERROR NAME MISSING FROM ID LIST

$row" . PHP_EOL; @@ -60,24 +57,21 @@ } fclose($handle); } -echo "all stats loaded successfuly"; foreach ($stats as $id => $stat) { echo $id . "
" . PHP_EOL; - $doc = $db->get($id); + $doc = $db->get($id); echo $doc->name . "
" . PHP_EOL; -// print_r($stat); + print_r($stat); + die(); // print_r($doc); $changed = false; if (!isset($doc->statistics)) { $changed = true; $doc->statistics = Array(); - } else { - $doc->statistics = object_to_array($doc->statistics); } foreach ($stat as $timePeriod => $value) { - if (!isset($doc->statistics["foiRequests"][$timePeriod]) - || $doc->statistics["foiRequests"][$timePeriod] != $value - ) { + if (!isset($doc->statistics->foiRequests->$timePeriod) + || $doc->statistics->foiRequests->$timePeriod != $value) { $changed = true; $doc->statistics["foiRequests"][$timePeriod] = $value; } @@ -87,7 +81,6 @@ } else { echo "not changed" . "
" . PHP_EOL; } -//print_r($doc);die(); } ?> --- a/admin/logo.svg +++ b/admin/logo.svg @@ -1,34 +1,58 @@ - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + --- a/admin/massdelete.sh +++ /dev/null @@ -1,10 +1,1 @@ -for line in `curl "http://localhost:5984/disclosr-foidocuments/_design/app/_view/byAgencyID?reduce=false&keys=%5B\"5716ce0aacfe98f7d638b7a66b7f1040\"%5D&limit=600" | xargs -L1`; do -# echo $line - id=`echo $line | grep -Po '_id:.*?[^\\\],' | perl -pe 's/_id://; s/^//; s/,$//'` - rev=`echo $line | grep -Po 'rev:.*?[^\\\],'| perl -pe 's/rev://; s/^//; s/,$//'` - if [ -n "$id" ]; then - echo "curl -X DELETE http://localhost:5984/disclosr-foidocuments/$id?rev=$rev" - curl -X DELETE http://localhost:5984/disclosr-foidocuments/$id?rev=$rev - fi -done; --- /dev/null +++ b/admin/neo4jimporter/.idea/.name @@ -1,1 +1,1 @@ - +neo4jimporter --- /dev/null +++ b/admin/neo4jimporter/.idea/compiler.xml @@ -1,1 +1,34 @@ + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/encodings.xml @@ -1,1 +1,6 @@ + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/libraries/Maven__com_googlecode_concurrentlinkedhashmap_concurrentlinkedhashmap_lru_1_3_1.xml @@ -1,1 +1,13 @@ - + + + + + + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/libraries/Maven__javax_xml_stream_stax_api_1_0_2.xml @@ -1,1 +1,13 @@ - + + + + + + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/libraries/Maven__org_apache_commons_commons_lang3_3_0_1.xml @@ -1,1 +1,13 @@ - + + + + + + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/libraries/Maven__org_apache_geronimo_specs_geronimo_jta_1_1_spec_1_1_1.xml @@ -1,1 +1,13 @@ - + + + + + + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/libraries/Maven__org_apache_lucene_lucene_core_3_6_2.xml @@ -1,1 +1,13 @@ - + + + + + + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/libraries/Maven__org_codehaus_woodstox_stax2_api_3_1_2.xml @@ -1,1 +1,13 @@ - + + + + + + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/libraries/Maven__org_codehaus_woodstox_woodstox_core_asl_4_2_0.xml @@ -1,1 +1,13 @@ - + + + + + + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/libraries/Maven__org_neo4j_neo4j_2_0_0_M03.xml @@ -1,1 +1,13 @@ - + + + + + + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/libraries/Maven__org_neo4j_neo4j_cypher_2_0_0_M03.xml @@ -1,1 +1,13 @@ - + + + + + + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/libraries/Maven__org_neo4j_neo4j_graph_algo_2_0_0_M03.xml @@ -1,1 +1,13 @@ - + + + + + + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/libraries/Maven__org_neo4j_neo4j_graph_matching_2_0_0_M03.xml @@ -1,1 +1,13 @@ - + + + + + + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/libraries/Maven__org_neo4j_neo4j_jmx_2_0_0_M03.xml @@ -1,1 +1,13 @@ - + + + + + + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/libraries/Maven__org_neo4j_neo4j_kernel_2_0_0_M03.xml @@ -1,1 +1,13 @@ - + + + + + + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/libraries/Maven__org_neo4j_neo4j_lucene_index_2_0_0_M03.xml @@ -1,1 +1,13 @@ - + + + + + + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/libraries/Maven__org_neo4j_neo4j_udc_2_0_0_M03.xml @@ -1,1 +1,13 @@ - + + + + + + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/libraries/Maven__org_scala_lang_scala_library_2_10_0.xml @@ -1,1 +1,13 @@ - + + + + + + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/libraries/Maven__postgresql_postgresql_9_1_901_jdbc4.xml @@ -1,1 +1,13 @@ - + + + + + + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/misc.xml @@ -1,1 +1,27 @@ + + + + + + + + + + + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/modules.xml @@ -1,1 +1,10 @@ + + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/neo4jimporter.iml @@ -1,1 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/scopes/scope_settings.xml @@ -1,1 +1,5 @@ - + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/vcs.xml @@ -1,1 +1,9 @@ + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/.idea/workspace.xml @@ -1,1 +1,560 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + localhost + 5050 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + file://$PROJECT_DIR$/src/main/java/StAXSample.java + 126 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Android 4.2 Platform + + + + + + + + neo4jimporter + + + + + + + + Android 4.2 Platform + + + + + + + + Maven: com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru:1.3.1 + + + + + + + + + --- /dev/null +++ b/admin/neo4jimporter/agency-sample.xml @@ -1,1 +1,9967 @@ + + + + CA 1 + 19001029 + Governor-General + + + + + + CA 1 + 19001029 + CEREMONIAL FUNCTIONS + 20071203 + (null) + (null) + + + CA 1 + 19001029 + COMMONWEALTH STATE RELATIONS + 20071203 + (null) + (null) + + + CA 1 + 19001029 + GOVERNOR GENERAL + 20071203 + (null) + (null) + + + CA 1 + (null) + 19140701 + (null) + PCS + (null) + + + CA 1 + (null) + 19001029 + (null) + PCS + (null) + + + CA 1 + (null) + 19820729 + (null) + PCS + (null) + + + CA 1 + (null) + 19350101 + (null) + PCS + circa + + + CA 1 + (null) + 19530501 + (null) + PCS + circa + + + CA 1 + (null) + 19751111 + (null) + PCS + (null) + + + CA 1 + (null) + 19310101 + (null) + PCS + circa + + + CA 1 + (null) + 19540601 + (null) + PCS + circa + + + CA 1 + (null) + 19540602 + (null) + PCS + (null) + + + CA 1 + (null) + 19100101 + (null) + PCS + (null) + + + CA 1 + (null) + 19370101 + (null) + PCS + (null) + + + CA 1 + (null) + 19200101 + (null) + PCS + (null) + + + CA 1 + (null) + 19740401 + (null) + PCS + (null) + + + CA 1 + (null) + 19370801 + (null) + PCS + (null) + + + CA 1 + (null) + 19100101 + (null) + PCS + (null) + + + CA 1 + (null) + 19370101 + (null) + PCS + (null) + + + CA 1 + (null) + 19250101 + (null) + PCS + (null) + + + CA 1 + (null) + 19260101 + (null) + PCS + (null) + + + CA 1 + (null) + 19390101 + (null) + PCS + (null) + + + CA 1 + (null) + 19380101 + (null) + PCS + (null) + + + CA 1 + (null) + (null) + (null) + PCS + (null) + + +