projects / disclosr.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
Merge branch 'master' of ssh://apples.lambdacomplex.org/git/disclosr
Merge branch 'master' of ssh://apples.lambdacomplex.org/git/disclosr

Conflicts:
admin/logo.svg

Former-commit-id: 850f69e8af3bbfb3467f5708676d7bf99fce2571

file:a/admin/importOAICFOIrequests.php -> file:b/admin/importOAICFOIrequests.php
<?php <?php
   
require_once '../include/common.inc.php'; require_once '../include/common.inc.php';
   
$db = $server->get_db('disclosr-agencies'); $db = $server->get_db('disclosr-agencies');
$rows = $db->get_view("app", "byName")->rows; $rows = $db->get_view("app", "byName")->rows;
$nametoid = Array(); $nametoid = Array();
$stats = Array(); $stats = Array();
foreach ($rows as $row) { foreach ($rows as $row) {
$nametoid[trim($row->key)] = $row->value; $nametoid[trim($row->key)] = $row->value;
} }
$row = 0; $row = 0;
$headers = Array(); $headers = Array();
// source: http://data.gov.au/dataset/freedom-of-information-quarterly-request-and-review-statistical-data-2011-12/ // source: http://data.gov.au/dataset/freedom-of-information-quarterly-request-and-review-statistical-data-2011-12/
if (($handle = fopen("FOI-quarterly-requests-and-reviews-2011-12.csv", "r")) !== FALSE) { if (($handle = fopen("FOI-quarterly-requests-and-reviews-2011-12.csv", "r")) !== FALSE) {
while (($data = fgetcsv($handle, 10000, ",")) !== FALSE) { while (($data = fgetcsv($handle, 10000, ",")) !== FALSE) {
if ($row >= 1) { if ($row >= 1) {
// print_r($data); // print_r($data);
$name = trim($data[2]); $name = trim($data[2]);
echo "$name <br>"; // echo "$name <br>";
if ($data[0] != "TOTALS" && $data[0] != "") { if ($data[0] != "TOTALS" && $data[0] != "") {
if (isset($nametoid[$name])) { if (isset($nametoid[$name])) {
$id = $nametoid[$name]; $id = $nametoid[$name];
$timePeriod = $data[0] . "-Q" . $data[1]; $timePeriod = $data[0] . "-Q" . $data[1];
   
echo "$timePeriod <br>"; // echo "$timePeriod <br>";
unset($data[0]); unset($data[0]);
unset($data[1]); unset($data[1]);
unset($data[2]); unset($data[2]);
unset($data[3]); unset($data[3]);
unset($data[4]); unset($data[4]);
unset($data[5]); unset($data[5]);
unset($data[6]); unset($data[6]);
unset($data[7]); unset($data[7]);
unset($data[8]); unset($data[8]);
   
//echo $id . "<br>" . PHP_EOL; //echo $id . "<br>" . PHP_EOL;
$result = Array("source" => "http://data.gov.au/dataset/freedom-of-information-quarterly-request-and-review-statistical-data-2011-12/"); $result = Array("source" => "http://data.gov.au/dataset/freedom-of-information-quarterly-request-and-review-statistical-data-2011-12/");
foreach ($data as $key => $datum) { foreach ($data as $key => $datum) {
if ($datum != 0) { if ($datum != 0) {
  // tODO prefix header with "FOI"
  if (isset($stats[$id][$timePeriod][$key])) $datum += $stats[$id][$timePeriod][$key];
$result[trim($headers[$key])] = $datum; $result[trim($headers[$key])] = $datum;
} }
} }
$stats[$id][$timePeriod] = $result; $stats[$id][$timePeriod] = $result;
  // TODO merge if already exists
//print_r($stats); //print_r($stats);
} else { } else {
echo "<br>ERROR NAME MISSING FROM ID LIST<br><bR> $row" . PHP_EOL; echo "<br>ERROR NAME MISSING FROM ID LIST<br><bR> $row" . PHP_EOL;
print_r($data); print_r($data);
die(); die();
} }
} }
} else { } else {
$headers = $data; $headers = $data;
//print_r($headers); //print_r($headers);
} }
$row++; $row++;
} }
fclose($handle); fclose($handle);
} }
  echo "all stats loaded successfuly";
foreach ($stats as $id => $stat) { foreach ($stats as $id => $stat) {
echo $id . "<br>" . PHP_EOL; echo $id . "<br>" . PHP_EOL;
$doc = $db->get($id); $doc = $db->get($id);
echo $doc->name . "<br>" . PHP_EOL; echo $doc->name . "<br>" . PHP_EOL;
print_r($stat); // print_r($stat);
die();  
// print_r($doc); // print_r($doc);
$changed = false; $changed = false;
if (!isset($doc->statistics)) { if (!isset($doc->statistics)) {
$changed = true; $changed = true;
$doc->statistics = Array(); $doc->statistics = Array();
  } else {
  $doc->statistics = object_to_array($doc->statistics);
} }
foreach ($stat as $timePeriod => $value) { foreach ($stat as $timePeriod => $value) {
if (!isset($doc->statistics->foiRequests->$timePeriod) if (!isset($doc->statistics["foiRequests"][$timePeriod])
|| $doc->statistics->foiRequests->$timePeriod != $value) { || $doc->statistics["foiRequests"][$timePeriod] != $value
  ) {
$changed = true; $changed = true;
$doc->statistics["foiRequests"][$timePeriod] = $value; $doc->statistics["foiRequests"][$timePeriod] = $value;
} }
} }
if ($changed) { if ($changed) {
$db->save($doc); $db->save($doc);
} else { } else {
echo "not changed" . "<br>" . PHP_EOL; echo "not changed" . "<br>" . PHP_EOL;
} }
  //print_r($doc);die();
} }
?> ?>
   
file:a/admin/logo.svg (deleted)
<?xml version="1.0" encoding="utf-8"?>  
<!-- Generator: Adobe Illustrator 15.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->  
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">  
<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"  
width="100px" height="100px" viewBox="0 0 100 100" enable-background="new 0 0 100 100" xml:space="preserve">  
<g id="docs">  
<path d="M86.107,11.001l5.229,9.952c0,0-1.832,0.083-5.297,1.95c-2.312,1.249-6.468,6.246-6.468,6.246l-7.745-11.148  
c0,0,4.2-3.996,6.513-5.242C81.805,10.889,86.107,11.001,86.107,11.001"/>  
<path d="M65.604,41.642l-3.151-9.868c0,0-5.44,3.56-8.017,4.074c-1.009,0.202-1.931,0.335-2.75,0.425L65.604,41.642z"/>  
<path d="M72.326,44.231c0.268-0.226,0.537-0.44,0.804-0.616c3.104-2.054,6.14-3.685,6.269-3.755l1.881-1.005l1.369,1.634  
l2.865,3.417l3.197-4.334L76.68,30.694l-8.74,11.847L72.326,44.231z"/>  
<path d="M39.918,31.734l4.825,1.86l3.33,0.212c0.04,0.001,0.269,0.015,0.652,0.015c0.91,0,2.798-0.072,5.196-0.551  
c1.427-0.284,5.007-2.332,7.093-3.695l2.889-1.888l1.05,3.285l2.496,7.812l5.889-7.985l-4.625,0.163l1.349-6.225l-14.928-3.233  
l-2.095,9.667c-0.531-2.599-1.841-5.727-1.841-5.727l-13.488,1.522c0,0,0.885,2.206,1.586,4.529L39.918,31.734z"/>  
<path d="M91.232,66.473c-1.102-0.691-2.322-1.143-3.414-1.434l-3.779,9.805c1.932,1.246,5.197,5.738,5.197,5.738l7.336-9.207  
C96.572,71.377,93.162,67.682,91.232,66.473z"/>  
<path d="M93.191,53.076l-3.655,1.225c-0.019,0.007-1.779,0.613-4.117,2.068l2.817,4.869l0.625,1.08  
c3.307-0.562,7.728-1.923,7.728-1.923l-2.332-15.261c0,0-2.934,1.277-5.853,2.221l2.318,2.766L93.191,53.076z"/>  
<path d="M79.271,46.91l0.865,0.334l0.459,0.801l3.504,6.05c2.646-1.636,4.61-2.287,4.61-2.287l-8.075-9.632  
c0,0-2.584,1.391-5.376,3.188L79.271,46.91z"/>  
</g>  
<g id="trunk">  
 
<radialGradient id="SVGID_1_" cx="66.0195" cy="72.8555" r="7.877" gradientTransform="matrix(1 0 0 2 0 -72.8555)" gradientUnits="userSpaceOnUse">  
<stop offset="0.6503" style="stop-color:#E61E24"/>  
<stop offset="1" style="stop-color:#9D1F1F"/>  
</radialGradient>  
<circle fill="url(#SVGID_1_)" cx="66.02" cy="72.855" r="7.877"/>  
 
<radialGradient id="SVGID_2_" cx="22.6929" cy="72.8555" r="7.877" gradientTransform="matrix(1 0 0 2 0 -72.8555)" gradientUnits="userSpaceOnUse">  
<stop offset="0.6503" style="stop-color:#E61E24"/>  
<stop offset="1" style="stop-color:#9D1F1F"/>  
</radialGradient>  
<circle fill="url(#SVGID_2_)" cx="22.693" cy="72.855" r="7.877"/>  
 
<radialGradient id="SVGID_3_" cx="15.4717" cy="59.7266" r="12.1478" gradientTransform="matrix(1 0 0 2 0 -59.7266)" gradientUnits="userSpaceOnUse">  
<stop offset="0.6503" style="stop-color:#E61E24"/>  
<stop offset="1" style="stop-color:#9D1F1F"/>  
</radialGradient>  
<path fill="url(#SVGID_3_)" d="M22.693,62.352c1.915,0,3.705,0.521,5.251,1.42V47.912H12.191L3,57.103v14.438h9.281  
C12.931,66.369,17.347,62.352,22.693,62.352z"/>  
 
<radialGradient id="SVGID_4_" cx="43.4023" cy="68.917" r="9.0473" gradientTransform="matrix(1 0 0 2 0 -68.917)" gradientUnits="userSpaceOnUse">  
<stop offset="0.6503" style="stop-color:#E61E24"/>  
<stop offset="1" style="stop-color:#9D1F1F"/>  
</radialGradient>  
<path fill="url(#SVGID_4_)" d="M55.925,69.986l-9.583-3.695H30.88c1.186,1.475,1.978,3.279,2.225,5.252h22.502  
C55.674,71.01,55.779,70.49,55.925,69.986z"/>  
 
<radialGradient id="SVGID_5_" cx="58.7637" cy="54.9512" r="25.2357" gradientTransform="matrix(1 0 0 2 0 -54.9512)" gradientUnits="userSpaceOnUse">  
<stop offset="0.6503" style="stop-color:#E61E24"/>  
<stop offset="1" style="stop-color:#9D1F1F"/>  
</radialGradient>  
<path fill="url(#SVGID_5_)" d="M79.312,48.238L39.961,33.06l-9.384,24.335l26.381,10.174c1.824-3.115,5.198-5.218,9.062-5.218  
c5.791,0,10.504,4.712,10.504,10.501c0,0.744-0.082,1.473-0.229,2.174l4.713,1.816l5.943-15.411L79.312,48.238z"/>  
</g>  
</svg>  
 
file:b/admin/massdelete.sh (new)
  for line in `curl "http://localhost:5984/disclosr-foidocuments/_design/app/_view/byAgencyID?reduce=false&keys=%5B\"5716ce0aacfe98f7d638b7a66b7f1040\"%5D&limit=600" | xargs -L1`; do
  # echo $line
  id=`echo $line | grep -Po '_id:.*?[^\\\],' | perl -pe 's/_id://; s/^//; s/,$//'`
  rev=`echo $line | grep -Po 'rev:.*?[^\\\],'| perl -pe 's/rev://; s/^//; s/,$//'`
  if [ -n "$id" ]; then
  echo "curl -X DELETE http://localhost:5984/disclosr-foidocuments/$id?rev=$rev"
  curl -X DELETE http://localhost:5984/disclosr-foidocuments/$id?rev=$rev
  fi
  done;
 
file:a/admin/neo4jimporter/pom.xml -> file:b/admin/neo4jimporter/pom.xml
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>org.lambdacomplex.contractdashboard</groupId> <groupId>org.lambdacomplex.contractdashboard</groupId>
<artifactId>neo4jimporter</artifactId> <artifactId>neo4jimporter</artifactId>
<version>0.0.1-SNAPSHOT</version> <version>0.0.1-SNAPSHOT</version>
<repositories> <repositories>
<repository> <repository>
<id>http://repository.codehaus.org/</id> <id>http://repository.codehaus.org/</id>
<url>http://repository.codehaus.org/</url> <url>http://repository.codehaus.org/</url>
</repository> </repository>
</repositories> </repositories>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>org.neo4j</groupId> <groupId>org.neo4j</groupId>
<artifactId>neo4j</artifactId> <artifactId>neo4j</artifactId>
<version>2.0.0-M03</version> <version>2.0.0-M03</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>postgresql</groupId> <groupId>postgresql</groupId>
<artifactId>postgresql</artifactId> <artifactId>postgresql</artifactId>
<version>9.1-901.jdbc4</version> <version>9.1-901.jdbc4</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.commons</groupId> <groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId> <artifactId>commons-lang3</artifactId>
<version>3.0.1</version> <version>3.0.1</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.codehaus.woodstox</groupId> <groupId>org.codehaus.woodstox</groupId>
<artifactId>woodstox-core-asl</artifactId> <artifactId>woodstox-core-asl</artifactId>
<version>4.2.0</version> <version>4.2.0</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.codehaus.woodstox</groupId> <groupId>org.codehaus.woodstox</groupId>
<artifactId>stax2-api</artifactId> <artifactId>stax2-api</artifactId>
<version>3.1.2</version> <version>3.1.2</version>
</dependency> </dependency>
   
</dependencies> </dependencies>
<build> <build>
<plugins> <plugins>
<plugin> <plugin>
<groupId>org.codehaus.mojo</groupId> <groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId> <artifactId>exec-maven-plugin</artifactId>
<version>1.2.1</version> <version>1.2.1</version>
<executions> <executions>
<execution> <execution>
<goals> <goals>
<goal>java</goal> <goal>java</goal>
</goals> </goals>
</execution> </execution>
</executions> </executions>
<configuration> <configuration>
<mainClass>Importer</mainClass> <mainClass>StAXSample</mainClass>
</configuration> </configuration>
</plugin> </plugin>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId> <artifactId>maven-compiler-plugin</artifactId>
<configuration> <configuration>
<source>1.6</source> <source>1.6</source>
<target>1.6</target> <target>1.6</target>
</configuration> </configuration>
</plugin> </plugin>
</plugins> </plugins>
</build> </build>
</project> </project>
   
file:a/admin/neo4jimporter/src/main/java/StAXSample.java -> file:b/admin/neo4jimporter/src/main/java/StAXSample.java
import org.neo4j.graphdb.DynamicLabel; import org.neo4j.graphdb.DynamicLabel;
import org.neo4j.graphdb.DynamicRelationshipType; import org.neo4j.graphdb.DynamicRelationshipType;
import org.neo4j.graphdb.Label; import org.neo4j.graphdb.Label;
import org.neo4j.unsafe.batchinsert.BatchInserter; import org.neo4j.unsafe.batchinsert.BatchInserter;
import org.neo4j.unsafe.batchinsert.BatchInserters; import org.neo4j.unsafe.batchinsert.BatchInserters;
   
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
   
import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLEventReader; import javax.xml.stream.XMLEventReader;
import javax.xml.stream.events.XMLEvent; import javax.xml.stream.events.XMLEvent;
   
public class StAXSample { public class StAXSample {
   
   
HashMap<String, Long> agencyIDs = new HashMap<String, Long>(); HashMap<String, Long> agencyIDs = new HashMap<String, Long>();
HashMap<String, Boolean> agencyFullVersion = new HashMap<String, Boolean>(); HashMap<String, Boolean> agencyFullVersion = new HashMap<String, Boolean>();
Label agencyLabel = DynamicLabel.label("Agency"); Label agencyLabel = DynamicLabel.label("Agency");
HashMap<String, Long> locationIDs = new HashMap<String, Long>(); HashMap<String, Long> locationIDs = new HashMap<String, Long>();
Label locationLabel = DynamicLabel.label("Location"); Label locationLabel = DynamicLabel.label("Location");
HashMap<String, Long> functionIDs = new HashMap<String, Long>(); HashMap<String, Long> functionIDs = new HashMap<String, Long>();
Label functionLabel = DynamicLabel.label("Function"); Label functionLabel = DynamicLabel.label("Function");
HashMap<String, Long> statusIDs = new HashMap<String, Long>(); HashMap<String, Long> statusIDs = new HashMap<String, Long>();
Label statusLabel = DynamicLabel.label("Location"); Label statusLabel = DynamicLabel.label("Location");
BatchInserter inserter; BatchInserter inserter;