From: Alex Sadleir <maxious@lambdacomplex.org>
Date: Wed, 02 Apr 2014 02:28:56 +0000
Subject: naa import update
X-Git-Url: http://maxious.lambdacomplex.org/git/?p=disclosr.git&a=commitdiff&h=c1289bb746febc08b38df8942d0ac0f452cfdd00
---
naa import update
---


--- a/admin/neo4jimporter/src/main/java/StAXSample.java
+++ b/admin/neo4jimporter/src/main/java/StAXSample.java
@@ -84,6 +84,44 @@
                     if (e.isStartElement()) {
                         if (hasStartTagName(e, "AGENCIES")) {
                             System.out.println("Agencies file loaded... ");
+                        } else if (hasStartTagName(e, "AGENCY_NO")) {
+                            previousAgency.put("agency_no", getCharacters(r));
+                        } else if (hasStartTagName(e, "TITLE")) {
+                            String title = getCharacters(r);
+                            previousAgency.put("name", title);
+                            previousAgency.put("label", title);
+                        } else if (hasStartTagName(e, "START_DATE")) {
+                            String start_date = getCharacters(r);
+                            if (start_date != null && !start_date.equals(" ") && !start_date.equals("(null)")) {
+                                previousAgency.put("start_date", Integer.parseInt(start_date));
+                            }
+                        } else if (hasStartTagName(e, "START_DATE_QUAL")) {
+                            previousAgency.put("start_date_qual", getCharacters(r));
+                        } else if (hasStartTagName(e, "ALTERNATIVE_TITLE")) {
+                            previousAgency.put("alternative_title", getCharacters(r));
+                        } else if (hasStartTagName(e, "END_DATE")) {
+                            String end_date = getCharacters(r);
+                            if (end_date != null && !end_date.equals(" ") && !end_date.equals("(null)")) {
+                                previousAgency.put("end_date", Integer.parseInt(end_date));
+                            }
+                        } else if (hasStartTagName(e, "END_DATE_QUAL")) {
+                            previousAgency.put("end_date_qual", getCharacters(r));
+                            // save agency
+                            getAgency(previousAgency);
+                            previousAgency = new HashMap<String, Object>();
+                        }
+                    }
+                }
+                r = xmlif.createXMLEventReader(
+                        filename,
+                        //new FileInputStream(new File(xmlFileURL.toURI())));
+                        new FileInputStream(new File(filename)));
+                while (r.hasNext()) {
+                    XMLEvent e = r.nextEvent();
+
+                    if (e.isStartElement()) {
+                        if (hasStartTagName(e, "AGENCIES")) {
+                            System.out.println("Agencies file loaded again... ");
                         } else if (hasStartTagName(e, "AGENCY_LINK")) {
                             processAgencyLink(r);
                         } else if (hasStartTagName(e, "AGENCY_LOCATION")) {
@@ -103,27 +141,6 @@
 Unexpected character 'C' (code 67) in start tag Expected a quote
  at [row,col,system-id]: [1093387,18,"agency-sample.xml"]
                              */
-                        } else if (hasStartTagName(e, "AGENCY_NO")) {
-                            previousAgency.put("agency_no", getCharacters(r));
-                        } else if (hasStartTagName(e, "TITLE")) {
-                            String title = getCharacters(r);
-                            previousAgency.put("name", title);
-                            previousAgency.put("label", title);
-                        } else if (hasStartTagName(e, "START_DATE")) {
-                            previousAgency.put("start_date", getCharacters(r));
-                        } else if (hasStartTagName(e, "START_DATE_QUAL")) {
-                            previousAgency.put("start_date_qual", getCharacters(r));
-                        } else if (hasStartTagName(e, "ALTERNATIVE_TITLE")) {
-                            previousAgency.put("alternative_title", getCharacters(r));
-                        } else if (hasStartTagName(e, "END_DATE")) {
-                            previousAgency.put("end_date", getCharacters(r));
-                        } else if (hasStartTagName(e, "END_DATE_QUAL")) {
-                            previousAgency.put("end_date_qual", getCharacters(r));
-                            // save agency
-                            getAgency(previousAgency);
-                            previousAgency = new HashMap<String, Object>();
-                        } else {
-                            System.out.println("Unhandled tag: " + getStartTagName(e) + " content:" + getCharacters(r));
                         }
                     }
                 }
@@ -146,11 +163,14 @@
     }
 
     private long getAgency(Map<String, Object> properties) {
+        if (properties.get("agency_no") == null || properties.get("agency_no") == "(null)" || properties.get("agency_no") == " ") {
+            return 0;
+        }
         if (agencyIDs.get(properties.get("agency_no").toString()) == null) {
             long agencyID = inserter.createNode(properties, agencyLabel);
-            if (properties.values().size() > 2) {
+            /*if (properties.values().size() > 1) {
                 agencyFullVersion.put(properties.get("agency_no").toString(), true);
-            }
+            } */
             agencyIDs.put(properties.get("agency_no").toString(), agencyID);
             //if (agencyID % 10 == 0) {
                 System.out.println("Agency #"+agencyID);
@@ -158,10 +178,10 @@
             return agencyID;
         } else {
             long agencyID = agencyIDs.get(properties.get("agency_no").toString());
-            if (properties.values().size() > 2 && agencyFullVersion.get(properties.get("agency_no")) == null) {
+            /*if (properties.values().size() > 1 && agencyFullVersion.get(properties.get("agency_no")) == null) {
                 inserter.setNodeProperties(agencyID, properties);
                 agencyFullVersion.put(properties.get("agency_no").toString(), true);
-            }
+            } */
             return agencyID;
         }
     }
@@ -235,29 +255,33 @@
                 if (hasEndTagName(e, "AGENCY_LINK")) {
 
                     //System.out.println("Finished processing link:  type = " + link_type+ "; from = " + agency_from_no + "; to = " + agency_to_no);
-                    long agencyFromID, agencyToID;
-                    Map<String, Object> agencyFromProperties = new HashMap<String, Object>();
-                    agencyFromProperties.put("agency_no",agency_from_no);
-                    agencyFromID = getAgency(agencyFromProperties);
-                    Map<String, Object> agencyToProperties = new HashMap<String, Object>();
-                    agencyToProperties.put("agency_no",agency_to_no);
-                    agencyToID = getAgency(agencyToProperties);
-                    Map<String, Object> relProperties = new HashMap<String, Object>();
-                    relProperties.put("link_type", link_type);
-                    relProperties.put("start_date", start_date);
-                    if (start_date_qual != null && !start_date_qual.equals("(null)")) {
-                        relProperties.put("start_date_qual", start_date_qual);
-                    }
-                    if (end_date != null && !end_date.equals("(null)")) {
-                        relProperties.put("end_date", end_date);
-                    }
-                    if (end_date_qual != null && !end_date_qual.equals("(null)")) {
-                        relProperties.put("end_date_qual", end_date_qual);
-                    }
-                    inserter.createRelationship(agencyFromID, agencyToID,
-                            DynamicRelationshipType.withName("IS_LINKED_TO"), relProperties);
-
-                    break;
+                    if (agency_to_no != null && !agency_to_no.equals("(null)")) {
+                        long agencyFromID, agencyToID;
+                        Map<String, Object> agencyFromProperties = new HashMap<String, Object>();
+                        agencyFromProperties.put("agency_from_no", agency_from_no);
+                        agencyFromID = getAgency(agencyFromProperties);
+                        Map<String, Object> agencyToProperties = new HashMap<String, Object>();
+                        agencyToProperties.put("agency_to_no", agency_to_no);
+                        agencyToID = getAgency(agencyToProperties);
+                        Map<String, Object> relProperties = new HashMap<String, Object>();
+                        relProperties.put("link_type", link_type);
+                        if (start_date != null && !start_date.equals("(null)")) {
+                            relProperties.put("start_date", Integer.parseInt(start_date));
+                        }
+                        if (start_date_qual != null && !start_date_qual.equals("(null)")) {
+                            relProperties.put("start_date_qual", start_date_qual);
+                        }
+                        if (end_date != null && !end_date.equals("(null)")) {
+                            relProperties.put("end_date", Integer.parseInt(end_date));
+                        }
+                        if (end_date_qual != null && !end_date_qual.equals("(null)")) {
+                            relProperties.put("end_date_qual", end_date_qual);
+                        }
+                        inserter.createRelationship(agencyFromID, agencyToID,
+                                DynamicRelationshipType.withName("IS_LINKED_TO"), relProperties);
+                    }
+                        break;
+
                 }
             }
         }
@@ -288,7 +312,7 @@
                     agencyProperties.put("agency_no",of);
                     agencyID = getAgency(agencyProperties);
                     Map<String, Object> relProperties = new HashMap<String, Object>();
-                    relProperties.put("date", date);
+                    relProperties.put("date", fixDate(date));
                     inserter.createRelationship(agencyID, locationID,
                             DynamicRelationshipType.withName("HAS_LOCATION"), relProperties);
 
@@ -323,7 +347,7 @@
                     agencyProperties.put("agency_no",of);
                     agencyID = getAgency(agencyProperties);
                     Map<String, Object> relProperties = new HashMap<String, Object>();
-                    relProperties.put("date", date);
+                    relProperties.put("date", fixDate(date));
                     inserter.createRelationship(agencyID, statusID,
                             DynamicRelationshipType.withName("HAS_STATUS"), relProperties);
 
@@ -367,12 +391,12 @@
                     agencyProperties.put("agency_no",agency);
                     agencyID = getAgency(agencyProperties);
                     Map<String, Object> relProperties = new HashMap<String, Object>();
-                    relProperties.put("start_date", start_date);
+                    relProperties.put("start_date", Integer.parseInt(start_date));
                     if (start_date_qual != null && !start_date_qual.equals("(null)")) {
                         relProperties.put("start_date_qual", start_date_qual);
                     }
                     if (end_date != null && !end_date.equals("(null)")) {
-                        relProperties.put("end_date", end_date);
+                        relProperties.put("end_date", Integer.parseInt(end_date));
                     }
                     if (end_date_qual != null && !end_date_qual.equals("(null)")) {
                         relProperties.put("end_date_qual", end_date_qual);
@@ -385,7 +409,14 @@
             }
         }
     }
-
+    private int fixDate(String date) {
+        String[] parts = date.split("-");
+        if (parts.length == 3) {
+            return Integer.parseInt(""+parts[2]+parts[1]+parts[0]);
+        } else {
+            return 0;
+        }
+    }
     private String getCharacters(XMLEventReader rdr) throws XMLStreamException {
         XMLEvent e = rdr.nextEvent();
         if (e.isCharacters()) {
@@ -399,10 +430,6 @@
         return e.asStartElement().getName().getLocalPart().toLowerCase().equals(name.toLowerCase());
     }
 
-    private String getStartTagName(XMLEvent e) {
-        return e.asStartElement().getName().getLocalPart();
-    }
-
     private boolean hasEndTagName(XMLEvent e, String name) {
         return e.asEndElement().getName().getLocalPart().toLowerCase().equals(name.toLowerCase());
     }

--- /dev/null
+++ b/admin/time.php
@@ -1,1 +1,164 @@
+<?php
 
+
+// https://github.com/jadell/neo4jphp
+spl_autoload_register(function ($className) {
+    $libPath = '../lib/neo4jphp/lib/';
+    $classFile = str_replace('\\',DIRECTORY_SEPARATOR,$className).'.php';
+    $classPath = $libPath.$classFile;
+    if (file_exists($classPath)) {
+        require($classPath);
+    }
+});
+
+$nodes = "";
+$edges = "";
+$description = "";
+$nodeList = Array();
+
+function add_node($node) {
+    global $nodes,$nodeList;
+    if (!in_array($node->getId(),$nodeList)) {
+        $start = $node->getProperty("start_date") . $node->getProperty("date");
+        $end = $node->getProperty("end_date");
+        $nodes.= "<node id='".urlencode($node->getId())."'". ' label="'.htmlentities($node->getProperty("name")).'" '
+            . ($start != ""? 'start="'.$start.'" ' : ""). ($end != ""? 'end="'.$end.'" ' : "").'>'
+            ."<viz:color b='".rand(0,255)."' g='".rand(0,255)."' r='".rand(0,255)."'/>"
+            ."</node>". PHP_EOL;
+        $nodeList[] = $node->getId();
+    }
+}
+
+function add_edge($rel) {
+    global $edges;
+    $start = $rel->getProperty("start_date");
+    $end = $rel->getProperty("end_date");
+    $edges.= "<edge id='".urlencode($rel->getId())."' source='".urlencode($rel->getStartNode()->getId())."' target='".urlencode($rel->getEndNode()->getId())."' ".($start != ""? 'start="'.$start.'" ' : ""). ($end != ""? 'end="'.$end.'" ' : "")."/>". PHP_EOL;
+
+}
+
+function expandNode($node) {
+    //global $description;
+    //$description .= ($description == ""? "" : " and ").$node->getProperty("name");
+    add_node($node);
+
+    foreach ($node->getProperties() as $key => $value) {
+        // echo "$key: $value\n";
+    }
+    foreach ($node->getRelationships() as $rel) {
+        //echo($rel->getStartNode()->getId()." -> ".$rel->getEndNode()->getId()."<br>");
+        add_edge($rel);
+        add_node($rel->getStartNode());
+        add_node($rel->getEndNode());
+    }
+}
+
+//$ids = (isset($_REQUEST['ids']) ? $_REQUEST['ids'] : "");
+
+// Connecting to the default port 7474 on localhost
+$client = new Everyman\Neo4j\Client();
+//$client = new Everyman\Neo4j\Client('192.168.1.127');
+//print_r($client->getServerInfo());
+
+//https://github.com/jadell/neo4jphp/wiki/Caching
+$plugin = new Everyman\Neo4j\Cache\Variable();
+$client->getEntityCache()->setCache($plugin);
+
+/*$memcached = new Memcached();
+$memcached->addServer('localhost', 11211);
+
+$plugin = new Everyman\Neo4j\Cache\Memcached($memcached);
+$client->getEntityCache()->setCache($plugin);*/
+
+/*$requests = explode(";",$ids);
+
+
+foreach ($requests as $request) {
+
+    // Array("type" => "path", from=>"1234", to=>"4321","options" => Array())
+
+    $parts = explode("-",$request);
+
+    $requestType = $parts[0];
+    $requestId = $parts[1];
+    if ($requestType == 'node') {
+        expandNode($client->getNode($requestId));
+    } else {
+        findNode($requestType,$requestId);
+    }
+}
+
+function findNode($type,$id) {
+    global $client;
+    $typeMapping = Array (
+        "agency" => Array("label" => "Agency", "id" => "agencyID")
+    );
+
+    $queryString =
+        "MATCH (n:".$typeMapping[$type]["label"].")".
+        "WHERE n.".$typeMapping[$type]["id"]." = {nodeId}".
+        "RETURN n";
+
+    $query = new Everyman\Neo4j\Cypher\Query($client, $queryString, array('nodeId' => $id));
+    $result = $query->getResultSet();
+
+    foreach ($result as $row) {
+        expandNode( $row[0]);
+
+    }
+
+}  */
+
+// https://github.com/jadell/neo4jphp/wiki/Paths
+
+/*  https://github.com/jadell/neo4jphp/wiki/Traversals
+ * $traversal = new Everyman\Neo4j\Traversal($client);
+$traversal->addRelationship('KNOWS', Relationship::DirectionOut)
+    ->setPruneEvaluator(Traversal::PruneNone)
+    ->setReturnFilter(Traversal::ReturnAll)
+    ->setMaxDepth(4);
+
+$nodes = $traversal->getResults($startNode, Traversal::ReturnTypeNode);
+
+ */
+
+/*https://github.com/jadell/neo4jphp/wiki/Cypher-and-gremlin-queries
+$queryString = "START n=node({nodeId}) ".
+ "MATCH (n)<-[:KNOWS]-(x)".
+ "WHERE x.name = {name}".
+ "RETURN x";
+$query = new Everyman\Neo4j\Cypher\Query($client, $queryString, array('nodeId' => 1, 'name' => 'Bob'));
+$result = $query->getResultSet();
+foreach ($result as $row) {
+ echo $row['x']->getProperty('name') . "\n";
+}*/
+
+$queryString =
+    "MATCH (n) WHERE has(n.start_date) and n.start_date < 18870000 RETURN n";
+  $description = '1887';
+$query = new Everyman\Neo4j\Cypher\Query($client, $queryString);
+$result = $query->getResultSet();
+
+foreach ($result as $row) {
+    expandNode( $row[0]);
+
+}
+
+if (!isset($_REQUEST['debug'])) {
+    header('Content-Type: application/gexf+xml');
+    header('Content-Disposition: attachment; filename="'.urlencode(str_replace(" ","_",strtolower($description))).'.gexf"');
+}
+echo '<?xml version="1.0" encoding="UTF-8"?>
+<gexf xmlns="http://www.gexf.net/1.2draft" xmlns:viz="http://www.gexf.net/1.2draft/viz" version="1.2">
+    <meta lastmodifieddate="2009-03-20">
+        <creator>lobbyist.disclosurelo.gs</creator>
+        <description>'. $description. '</description>
+    </meta>
+    <graph mode="dynamic" defaultedgetype="directed" timeformat="date">
+
+        <nodes>'. $nodes. '</nodes>
+        <edges>'. $edges.' </edges>
+    </graph>
+</gexf>'. PHP_EOL;
+
+?>

--- /dev/null
+++ b/admin/timeline.php
@@ -1,1 +1,149 @@
+<?php
+// START n=node(*) OPTIONAL MATCH n<-[r]-parent WITH n, count(parent) AS parents WHERE parents = 0 RETURN n, parents;
+// no parents
 
+// START n=node(*) OPTIONAL MATCH (n:Agency)-[r]->(child:Agency) WITH n, count(child) AS children WHERE children = 0 AND n:Agency RETURN n, children;
+// no children
+
+// MATCH (n) WHERE has(n.start_date) and n.start_date < 18870000 RETURN n
+// time slice
+if (isset($_REQUEST['json'])) {
+
+   /* echo '{
+    "timeline":
+    {
+        "headline":"The Main Timeline Headline Goes here",
+        "type":"default",
+        "text":"<p>Intro body text goes here, some HTML is ok</p>",
+        "asset": {
+            "media":"http://yourdomain_or_socialmedialink_goes_here.jpg",
+            "credit":"Credit Name Goes Here",
+            "caption":"Caption text goes here"
+        },
+        "date": [
+            {
+                "startDate":"2011,12,10",
+                "endDate":"2011,12,11",
+                "headline":"Headline Goes Here",
+                "text":"<p>Body text goes here, some HTML is OK</p>",
+                "tag":"This is Optional",
+                "classname":"optionaluniqueclassnamecanbeaddedhere",
+                "asset": {
+                    "media":"http://twitter.com/ArjunaSoriano/status/164181156147900416",
+                    "thumbnail":"optional-32x32px.jpg",
+                    "credit":"Credit Name Goes Here",
+                    "caption":"Caption text goes here"
+                }
+            }
+        ],
+        "era": [
+            {
+                "startDate":"2011,12,10",
+                "endDate":"2011,12,11",
+                "headline":"Headline Goes Here",
+                "text":"<p>Body text goes here, some HTML is OK</p>",
+                "tag":"This is Optional"
+            }
+
+        ]
+    }
+}';   */
+
+    // https://github.com/jadell/neo4jphp
+    spl_autoload_register(function ($className) {
+        $libPath = '../lib/neo4jphp/lib/';
+        $classFile = str_replace('\\',DIRECTORY_SEPARATOR,$className).'.php';
+        $classPath = $libPath.$classFile;
+        if (file_exists($classPath)) {
+            require($classPath);
+        }
+    });
+
+
+// Connecting to the default port 7474 on localhost
+    $client = new Everyman\Neo4j\Client();
+//$client = new Everyman\Neo4j\Client('192.168.1.127');
+//print_r($client->getServerInfo());
+
+
+
+
+    //https://github.com/jadell/neo4jphp/wiki/Cypher-and-gremlin-queries
+    $queryString = "MATCH (n) where has(n.agency_no) RETURN n LIMIT 1000";
+    $query = new Everyman\Neo4j\Cypher\Query($client, $queryString);
+    $dates = Array();
+    $result = $query->getResultSet();
+    foreach ($result as $row) {
+     //$dates[] = Array("startDate" =>"2011,12,10",   "endDate"=>"2011,12,11",        "headline"=>"Headline Goes Here",        "text"=>"<p>Body text goes here, some HTML is OK</p>");
+        $name = $row['x']->getProperty('name');
+        $startDate = $row['x']->getProperty('start_date');
+        $startDate = substr_replace($startDate, ",", 6, 0);
+        $startDate = substr_replace($startDate, ",", 4, 0);
+        $endDate = $row['x']->getProperty('end_date');
+        if ($endDate == " ") $endDate = "20140101";
+        $endDate = substr_replace($endDate, ",", 6, 0);
+        $endDate = substr_replace($endDate, ",", 4, 0);
+
+        $dates[] = Array("startDate" =>$startDate,   "endDate"=>$endDate,        "headline"=>$name,        "text"=>"<p>Body text goes here, some HTML is OK</p>");
+
+    }
+
+
+
+    $timeline = Array( "headline"=>"The Main Timeline Headline Goes here",
+        "type"=>"default",
+        "text"=>"<p>Intro body text goes here, some HTML is ok</p>","date"=>$dates);
+    echo json_encode(Array("timeline" => $timeline));
+    die();
+}
+?>
+
+     <!DOCTYPE html>
+     <html lang="en"><!--
+
+  	88888888888 d8b                        888 d8b                888888   d8888b
+  	    888     Y8P                        888 Y8P                   88b d88P  Y88b
+  	    888                                888                       888 Y88b
+  	    888     888 88888b d88b     d88b   888 888 88888b     d88b   888   Y888b
+  	    888     888 888  888  88b d8P  Y8b 888 888 888  88b d8P  Y8b 888      Y88b
+  	    888     888 888  888  888 88888888 888 888 888  888 88888888 888        888
+  	    888     888 888  888  888 Y8b      888 888 888  888 Y8b      88P Y88b  d88P
+  	    888     888 888  888  888   Y8888  888 888 888  888   Y8888  888   Y8888P
+  	                                                                d88P
+  	                                                              d88P
+  	                                                            888P
+  	 -->
+     <head>
+         <title>Revolutionary User Interfaces</title>
+         <meta name="description" content="The human computer interface helps to define computing at any one time.">
+         <meta charset="utf-8">
+         <meta name="apple-mobile-web-app-capable" content="yes">
+         <meta name="apple-touch-fullscreen" content="yes">
+         <!-- Style-->
+         <style>
+             html, body {
+                 height:100%;
+                 padding: 0px;
+                 margin: 0px;
+             }
+         </style>
+         <link rel="stylesheet" type="text/css" href="http://cdn.knightlab.com/libs/timeline/latest/css/timeline.css">
+         <!-- HTML5 shim, for IE6-8 support of HTML elements--><!--[if lt IE 9]>
+         <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script><![endif]-->
+     </head>
+     <body>
+     <!-- BEGIN Timeline Embed -->
+     <div id="timeline-embed"></div>
+     <script type="text/javascript">
+         var timeline_config = {
+             width: "100%",
+             height: "100%",
+             source: 'timeline.php?json=yes',
+             font:	'SansitaOne-Kameron'
+         }
+     </script>
+     <script type="text/javascript" src="http://cdn.knightlab.com/libs/timeline/latest/js/storyjs-embed.js"></script>
+     <!-- END Timeline Embed-->
+
+     </body>
+     </html>