projects / disclosr.git / commitdiff
search:
summary | shortlog | log | commit | commitdiff | tree
unified | plain
naa import update
naa import update

file:a/admin/neo4jimporter/src/main/java/StAXSample.java -> file:b/admin/neo4jimporter/src/main/java/StAXSample.java
import org.neo4j.graphdb.DynamicLabel; import org.neo4j.graphdb.DynamicLabel;
import org.neo4j.graphdb.DynamicRelationshipType; import org.neo4j.graphdb.DynamicRelationshipType;
import org.neo4j.graphdb.Label; import org.neo4j.graphdb.Label;
import org.neo4j.unsafe.batchinsert.BatchInserter; import org.neo4j.unsafe.batchinsert.BatchInserter;
import org.neo4j.unsafe.batchinsert.BatchInserters; import org.neo4j.unsafe.batchinsert.BatchInserters;
   
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
   
import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLEventReader; import javax.xml.stream.XMLEventReader;
import javax.xml.stream.events.XMLEvent; import javax.xml.stream.events.XMLEvent;
   
public class StAXSample { public class StAXSample {
   
   
HashMap<String, Long> agencyIDs = new HashMap<String, Long>(); HashMap<String, Long> agencyIDs = new HashMap<String, Long>();
HashMap<String, Boolean> agencyFullVersion = new HashMap<String, Boolean>(); HashMap<String, Boolean> agencyFullVersion = new HashMap<String, Boolean>();
Label agencyLabel = DynamicLabel.label("Agency"); Label agencyLabel = DynamicLabel.label("Agency");
HashMap<String, Long> locationIDs = new HashMap<String, Long>(); HashMap<String, Long> locationIDs = new HashMap<String, Long>();
Label locationLabel = DynamicLabel.label("Location"); Label locationLabel = DynamicLabel.label("Location");
HashMap<String, Long> functionIDs = new HashMap<String, Long>(); HashMap<String, Long> functionIDs = new HashMap<String, Long>();
Label functionLabel = DynamicLabel.label("Function"); Label functionLabel = DynamicLabel.label("Function");
HashMap<String, Long> statusIDs = new HashMap<String, Long>(); HashMap<String, Long> statusIDs = new HashMap<String, Long>();
Label statusLabel = DynamicLabel.label("Location"); Label statusLabel = DynamicLabel.label("Location");
BatchInserter inserter; BatchInserter inserter;
   
private String filename; private String filename;
   
public StAXSample() { public StAXSample() {
} }
   
public static void main(String[] args) { public static void main(String[] args) {
/*if (args.length != 1) { /*if (args.length != 1) {
System.out.println("Usage: StAXSample file.xml"); System.out.println("Usage: StAXSample file.xml");
System.exit(-1); System.exit(-1);
} */ } */
   
StAXSample ss = new StAXSample(); StAXSample ss = new StAXSample();
//ss.setFilename(args[0]); //ss.setFilename(args[0]);
ss.setFilename("agency-sample.xml"); ss.setFilename("agency-sample.xml");
ss.run(); ss.run();
} }
   
public void run() { public void run() {
   
Map<String, String> config = new HashMap<String, String>(); Map<String, String> config = new HashMap<String, String>();
config.put("neostore.nodestore.db.mapped_memory", "90M"); config.put("neostore.nodestore.db.mapped_memory", "90M");
inserter = BatchInserters.inserter("target/batchinserter-example-config", config); inserter = BatchInserters.inserter("target/batchinserter-example-config", config);
inserter.createDeferredSchemaIndex(agencyLabel).on("agency_no"); inserter.createDeferredSchemaIndex(agencyLabel).on("agency_no");
inserter.createDeferredSchemaIndex(locationLabel).on("location_name"); inserter.createDeferredSchemaIndex(locationLabel).on("location_name");
inserter.createDeferredSchemaIndex(functionLabel).on("thesaurus_term"); inserter.createDeferredSchemaIndex(functionLabel).on("thesaurus_term");
inserter.createDeferredSchemaIndex(statusLabel).on("status_name"); inserter.createDeferredSchemaIndex(statusLabel).on("status_name");
   
try { try {
XMLInputFactory xmlif = XMLInputFactory.newInstance(); XMLInputFactory xmlif = XMLInputFactory.newInstance();
xmlif.setProperty( xmlif.setProperty(
XMLInputFactory.IS_REPLACING_ENTITY_REFERENCES, XMLInputFactory.IS_REPLACING_ENTITY_REFERENCES,
Boolean.TRUE); Boolean.TRUE);
xmlif.setProperty( xmlif.setProperty(
XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES,
Boolean.FALSE); Boolean.FALSE);
//set the IS_COALESCING property to true //set the IS_COALESCING property to true
//to get whole text data as one event. //to get whole text data as one event.
xmlif.setProperty(XMLInputFactory.IS_COALESCING, Boolean.TRUE); xmlif.setProperty(XMLInputFactory.IS_COALESCING, Boolean.TRUE);
   
try { try {
XMLEventReader r = null; XMLEventReader r = null;
r = xmlif.createXMLEventReader( r = xmlif.createXMLEventReader(
filename, filename,
//new FileInputStream(new File(xmlFileURL.toURI()))); //new FileInputStream(new File(xmlFileURL.toURI())));
new FileInputStream(new File(filename))); new FileInputStream(new File(filename)));
   
   
//iterate as long as there are more events on the input stream //iterate as long as there are more events on the input stream
Map<String, Object> previousAgency = new HashMap<String, Object>(); Map<String, Object> previousAgency = new HashMap<String, Object>();
while (r.hasNext()) { while (r.hasNext()) {
XMLEvent e = r.nextEvent(); XMLEvent e = r.nextEvent();
   
if (e.isStartElement()) { if (e.isStartElement()) {
if (hasStartTagName(e, "AGENCIES")) { if (hasStartTagName(e, "AGENCIES")) {
System.out.println("Agencies file loaded... "); System.out.println("Agencies file loaded... ");
  } else if (hasStartTagName(e, "AGENCY_NO")) {
  previousAgency.put("agency_no", getCharacters(r));
  } else if (hasStartTagName(e, "TITLE")) {
  String title = getCharacters(r);
  previousAgency.put("name", title);
  previousAgency.put("label", title);
  } else if (hasStartTagName(e, "START_DATE")) {
  String start_date = getCharacters(r);
  if (start_date != null && !start_date.equals(" ") && !start_date.equals("(null)")) {
  previousAgency.put("start_date", Integer.parseInt(start_date));
  }
  } else if (hasStartTagName(e, "START_DATE_QUAL")) {
  previousAgency.put("start_date_qual", getCharacters(r));
  } else if (hasStartTagName(e, "ALTERNATIVE_TITLE")) {
  previousAgency.put("alternative_title", getCharacters(r));
  } else if (hasStartTagName(e, "END_DATE")) {
  String end_date = getCharacters(r);
  if (end_date != null && !end_date.equals(" ") && !end_date.equals("(null)")) {
  previousAgency.put("end_date", Integer.parseInt(end_date));
  }
  } else if (hasStartTagName(e, "END_DATE_QUAL")) {
  previousAgency.put("end_date_qual", getCharacters(r));
  // save agency
  getAgency(previousAgency);
  previousAgency = new HashMap<String, Object>();
  }
  }
  }
  r = xmlif.createXMLEventReader(
  filename,
  //new FileInputStream(new File(xmlFileURL.toURI())));
  new FileInputStream(new File(filename)));
  while (r.hasNext()) {
  XMLEvent e = r.nextEvent();
   
  if (e.isStartElement()) {
  if (hasStartTagName(e, "AGENCIES")) {
  System.out.println("Agencies file loaded again... ");
} else if (hasStartTagName(e, "AGENCY_LINK")) { } else if (hasStartTagName(e, "AGENCY_LINK")) {
processAgencyLink(r); processAgencyLink(r);
} else if (hasStartTagName(e, "AGENCY_LOCATION")) { } else if (hasStartTagName(e, "AGENCY_LOCATION")) {
processAgencyLocation(r); processAgencyLocation(r);
} else if (hasStartTagName(e, "AGENCY_FUNCTION")) { } else if (hasStartTagName(e, "AGENCY_FUNCTION")) {
processAgencyFunction(r); processAgencyFunction(r);
} else if (hasStartTagName(e, "AGENCY_STATUS")) { } else if (hasStartTagName(e, "AGENCY_STATUS")) {
processAgencyStatus(r); processAgencyStatus(r);
/* TODO /* TODO
Unhandled tag: AGENCY_NOTE content: Unhandled tag: AGENCY_NOTE content:
   
Unhandled tag: NOTE_AGENCY_NO content:CA 4886 Unhandled tag: NOTE_AGENCY_NO content:CA 4886
Unhandled tag: NOTE_TYPE content:Archivists note Unhandled tag: NOTE_TYPE content:Archivists note
Unhandled tag: NOTE content:null Unhandled tag: NOTE content:null
Unhandled tag: head content: Unhandled tag: head content:
   
Unexpected character 'C' (code 67) in start tag Expected a quote Unexpected character 'C' (code 67) in start tag Expected a quote
at [row,col,system-id]: [1093387,18,"agency-sample.xml"] at [row,col,system-id]: [1093387,18,"agency-sample.xml"]
*/ */
} else if (hasStartTagName(e, "AGENCY_NO")) {  
previousAgency.put("agency_no", getCharacters(r));  
} else if (hasStartTagName(e, "TITLE")) {  
String title = getCharacters(r);  
previousAgency.put("name", title);  
previousAgency.put("label", title);  
} else if (hasStartTagName(e, "START_DATE")) {  
previousAgency.put("start_date", getCharacters(r));  
} else if (hasStartTagName(e, "START_DATE_QUAL")) {  
previousAgency.put("start_date_qual", getCharacters(r));  
} else if (hasStartTagName(e, "ALTERNATIVE_TITLE")) {  
previousAgency.put("alternative_title", getCharacters(r));  
} else if (hasStartTagName(e, "END_DATE")) {  
previousAgency.put("end_date", getCharacters(r));  
} else if (hasStartTagName(e, "END_DATE_QUAL")) {  
previousAgency.put("end_date_qual", getCharacters(r));  
// save agency  
getAgency(previousAgency);  
previousAgency = new HashMap<String, Object>();  
} else {  
System.out.println("Unhandled tag: " + getStartTagName(e) + " content:" + getCharacters(r));  
} }
} }
} }
} catch (XMLStreamException ex) { } catch (XMLStreamException ex) {
System.out.println(ex.getMessage()); System.out.println(ex.getMessage());
   
if (ex.getNestedException() != null) { if (ex.getNestedException() != null) {
ex.getNestedException().printStackTrace(); ex.getNestedException().printStackTrace();
} }
} }
   
} catch (FileNotFoundException ex) { } catch (FileNotFoundException ex) {
System.err.println("Error. Cannot find \"" + filename + "\" in classpath."); System.err.println("Error. Cannot find \"" + filename + "\" in classpath.");
ex.printStackTrace(); ex.printStackTrace();
} catch (Exception ex) { } catch (Exception ex) {
ex.printStackTrace(); ex.printStackTrace();
} }
   
inserter.shutdown(); inserter.shutdown();
} }
   
private long getAgency(Map<String, Object> properties) { private long getAgency(Map<String, Object> properties) {
  if (properties.get("agency_no") == null || properties.get("agency_no") == "(null)" || properties.get("agency_no") == " ") {
  return 0;
  }
if (agencyIDs.get(properties.get("agency_no").toString()) == null) { if (agencyIDs.get(properties.get("agency_no").toString()) == null) {
long agencyID = inserter.createNode(properties, agencyLabel); long agencyID = inserter.createNode(properties, agencyLabel);
if (properties.values().size() > 2) { /*if (properties.values().size() > 1) {
agencyFullVersion.put(properties.get("agency_no").toString(), true); agencyFullVersion.put(properties.get("agency_no").toString(), true);
} } */
agencyIDs.put(properties.get("agency_no").toString(), agencyID); agencyIDs.put(properties.get("agency_no").toString(), agencyID);
//if (agencyID % 10 == 0) { //if (agencyID % 10 == 0) {
System.out.println("Agency #"+agencyID); System.out.println("Agency #"+agencyID);
//} //}
return agencyID; return agencyID;
} else { } else {
long agencyID = agencyIDs.get(properties.get("agency_no").toString()); long agencyID = agencyIDs.get(properties.get("agency_no").toString());
if (properties.values().size() > 2 && agencyFullVersion.get(properties.get("agency_no")) == null) { /*if (properties.values().size() > 1 && agencyFullVersion.get(properties.get("agency_no")) == null) {
inserter.setNodeProperties(agencyID, properties); inserter.setNodeProperties(agencyID, properties);
agencyFullVersion.put(properties.get("agency_no").toString(), true); agencyFullVersion.put(properties.get("agency_no").toString(), true);
} } */
return agencyID; return agencyID;
} }
} }
   
private long getLocation(String locationName) { private long getLocation(String locationName) {
if (locationIDs.get(locationName) == null) { if (locationIDs.get(locationName) == null) {
HashMap properties = new HashMap< String,Object > (); HashMap properties = new HashMap< String,Object > ();
properties.put("name", locationName); properties.put("name", locationName);
properties.put("label", locationName); properties.put("label", locationName);
long locationID = inserter.createNode(properties, locationLabel); long locationID = inserter.createNode(properties, locationLabel);
locationIDs.put(locationName, locationID); locationIDs.put(locationName, locationID);
return locationID; return locationID;
} else { } else {
return locationIDs.get(locationName); return locationIDs.get(locationName);
} }
} }
private long getFunction(String functionName) { private long getFunction(String functionName) {
if (functionIDs.get(functionName) == null) { if (functionIDs.get(functionName) == null) {
HashMap properties = new HashMap< String,Object > (); HashMap properties = new HashMap< String,Object > ();
properties.put("name", functionName); properties.put("name", functionName);
properties.put("label", functionName); properties.put("label", functionName);
long functionID = inserter.createNode(properties, functionLabel); long functionID = inserter.createNode(properties, functionLabel);
functionIDs.put(functionName, functionID); functionIDs.put(functionName, functionID);
return functionID; return functionID;
} else { } else {
return functionIDs.get(functionName); return functionIDs.get(functionName);
} }
} }
private long getStatus(String statusName) { private long getStatus(String statusName) {
if (statusIDs.get(statusName) == null) { if (statusIDs.get(statusName) == null) {
HashMap properties = new HashMap< String,Object > (); HashMap properties = new HashMap< String,Object > ();
properties.put("name", statusName); properties.put("name", statusName);
properties.put("label", statusName); properties.put("label", statusName);
long statusID = inserter.createNode(properties, statusLabel); long statusID = inserter.createNode(properties, statusLabel);
statusIDs.put(statusName, statusID); statusIDs.put(statusName, statusID);
return statusID; return statusID;
} else { } else {
return statusIDs.get(statusName); return statusIDs.get(statusName);
} }
} }
   
private void processAgencyLink(XMLEventReader rdr) throws Exception { private void processAgencyLink(XMLEventReader rdr) throws Exception {
String agency_from_no = null; String agency_from_no = null;
String agency_to_no = null; String agency_to_no = null;
String link_type = null; String link_type = null;
String start_date = null; String start_date = null;
String start_date_qual = null; String start_date_qual = null;
String end_date = null; String end_date = null;
String end_date_qual = null; String end_date_qual = null;
   
while (rdr.hasNext()) { while (rdr.hasNext()) {
XMLEvent e = rdr.nextEvent(); XMLEvent e = rdr.nextEvent();
if (e.isStartElement()) { if (e.isStartElement()) {
if (hasStartTagName(e, "LINK_AGENCY_NO")) { if (hasStartTagName(e, "LINK_AGENCY_NO")) {
agency_from_no = getCharacters(rdr); agency_from_no = getCharacters(rdr);
} else if (hasStartTagName(e, "LINK_TO_AGENCY_NO")) { } else if (hasStartTagName(e, "LINK_TO_AGENCY_NO")) {
agency_to_no = getCharacters(rdr); agency_to_no = getCharacters(rdr);
} else if (hasStartTagName(e, "LINK_TYPE")) { } else if (hasStartTagName(e, "LINK_TYPE")) {
link_type = getCharacters(rdr); link_type = getCharacters(rdr);
} else if (hasStartTagName(e, "START_DATE")) { } else if (hasStartTagName(e, "START_DATE")) {
start_date = getCharacters(rdr); start_date = getCharacters(rdr);
}else if (hasStartTagName(e, "START_DATE_QUAL")) { }else if (hasStartTagName(e, "START_DATE_QUAL")) {
start_date_qual = getCharacters(rdr); start_date_qual = getCharacters(rdr);
}else if (hasStartTagName(e, "END_DATE")) { }else if (hasStartTagName(e, "END_DATE")) {
end_date = getCharacters(rdr); end_date = getCharacters(rdr);
}else if (hasStartTagName(e, "END_DATE_QUAL")) { }else if (hasStartTagName(e, "END_DATE_QUAL")) {
end_date_qual = getCharacters(rdr); end_date_qual = getCharacters(rdr);
} }
} }
if (e.isEndElement()) { if (e.isEndElement()) {
if (hasEndTagName(e, "AGENCY_LINK")) { if (hasEndTagName(e, "AGENCY_LINK")) {
   
//System.out.println("Finished processing link: type = " + link_type+ "; from = " + agency_from_no + "; to = " + agency_to_no); //System.out.println("Finished processing link: type = " + link_type+ "; from = " + agency_from_no + "; to = " + agency_to_no);
long agencyFromID, agencyToID; if (agency_to_no != null && !agency_to_no.equals("(null)")) {
Map<String, Object> agencyFromProperties = new HashMap<String, Object>(); long agencyFromID, agencyToID;
agencyFromProperties.put("agency_no",agency_from_no); Map<String, Object> agencyFromProperties = new HashMap<String, Object>();
agencyFromID = getAgency(agencyFromProperties); agencyFromProperties.put("agency_from_no", agency_from_no);
Map<String, Object> agencyToProperties = new HashMap<String, Object>(); agencyFromID = getAgency(agencyFromProperties);
agencyToProperties.put("agency_no",agency_to_no); Map<String, Object> agencyToProperties = new HashMap<String, Object>();
agencyToID = getAgency(agencyToProperties); agencyToProperties.put("agency_to_no", agency_to_no);
Map<String, Object> relProperties = new HashMap<String, Object>(); agencyToID = getAgency(agencyToProperties);
relProperties.put("link_type", link_type); Map<String, Object> relProperties = new HashMap<String, Object>();
relProperties.put("start_date", start_date); relProperties.put("link_type", link_type);
if (start_date_qual != null && !start_date_qual.equals("(null)")) { if (start_date != null && !start_date.equals("(null)")) {
relProperties.put("start_date_qual", start_date_qual); relProperties.put("start_date", Integer.parseInt(start_date));
} }
if (end_date != null && !end_date.equals("(null)")) { if (start_date_qual != null && !start_date_qual.equals("(null)")) {
relProperties.put("end_date", end_date); relProperties.put("start_date_qual", start_date_qual);
} }
if (end_date_qual != null && !end_date_qual.equals("(null)")) { if (end_date != null && !end_date.equals("(null)")) {
relProperties.put("end_date_qual", end_date_qual); relProperties.put("end_date", Integer.parseInt(end_date));
} }
inserter.createRelationship(agencyFromID, agencyToID, if (end_date_qual != null && !end_date_qual.equals("(null)")) {
DynamicRelationshipType.withName("IS_LINKED_TO"), relProperties); relProperties.put("end_date_qual", end_date_qual);
  }
break; inserter.createRelationship(agencyFromID, agencyToID,
  DynamicRelationshipType.withName("IS_LINKED_TO"), relProperties);
  }
  break;
   
} }
} }
} }
} }
   
private void processAgencyLocation(XMLEventReader rdr) throws Exception { private void processAgencyLocation(XMLEventReader rdr) throws Exception {
String of = null; String of = null;
String name = null; String name = null;
String date = null; String date = null;
   
while (rdr.hasNext()) { while (rdr.hasNext()) {
XMLEvent e = rdr.nextEvent(); XMLEvent e = rdr.nextEvent();
if (e.isStartElement()) { if (e.isStartElement()) {
if (hasStartTagName(e, "LOCATION_AGENCY_NO")) { if (hasStartTagName(e, "LOCATION_AGENCY_NO")) {
of = getCharacters(rdr); of = getCharacters(rdr);
} else if (hasStartTagName(e, "LOCATION_TEXT")) { } else if (hasStartTagName(e, "LOCATION_TEXT")) {
name = getCharacters(rdr); name = getCharacters(rdr);
} else if (hasStartTagName(e, "LOCATION_DATE")) { } else if (hasStartTagName(e, "LOCATION_DATE")) {
date = getCharacters(rdr); date = getCharacters(rdr);
} }
} }
if (e.isEndElement()) { if (e.isEndElement()) {
if (hasEndTagName(e, "AGENCY_LOCATION")) { if (hasEndTagName(e, "AGENCY_LOCATION")) {
//System.out.println("Finished processing location: Name = " + name + "; of = " + of + "; date = " + date); //System.out.println("Finished processing location: Name = " + name + "; of = " + of + "; date = " + date);
long locationID, agencyID; long locationID, agencyID;
locationID = getLocation(name); locationID = getLocation(name);
Map<String, Object> agencyProperties = new HashMap<String, Object>(); Map<String, Object> agencyProperties = new HashMap<String, Object>();
agencyProperties.put("agency_no",of); agencyProperties.put("agency_no",of);
agencyID = getAgency(agencyProperties); agencyID = getAgency(agencyProperties);
Map<String, Object> relProperties = new HashMap<String, Object>(); Map<String, Object> relProperties = new HashMap<String, Object>();
relProperties.put("date", date); relProperties.put("date", fixDate(date));
inserter.createRelationship(agencyID, locationID, inserter.createRelationship(agencyID, locationID,
DynamicRelationshipType.withName("HAS_LOCATION"), relProperties); DynamicRelationshipType.withName("HAS_LOCATION"), relProperties);
   
break; break;
} }
} }
} }
} }
   
private void processAgencyStatus(XMLEventReader rdr) throws Exception { private void processAgencyStatus(XMLEventReader rdr) throws Exception {
String of = null; String of = null;
String status = null; String status = null;
String date = null; String date = null;
   
while (rdr.hasNext()) { while (rdr.hasNext()) {
XMLEvent e = rdr.nextEvent(); XMLEvent e = rdr.nextEvent();
if (e.isStartElement()) { if (e.isStartElement()) {
if (hasStartTagName(e, "STATUS_AGENCY_NO")) { if (hasStartTagName(e, "STATUS_AGENCY_NO")) {
of = getCharacters(rdr); of = getCharacters(rdr);
} else if (hasStartTagName(e, "STATUS")) { } else if (hasStartTagName(e, "STATUS")) {
status = getCharacters(rdr); status = getCharacters(rdr);
} else if (hasStartTagName(e, "STATUS_DATE")) { } else if (hasStartTagName(e, "STATUS_DATE")) {
date = getCharacters(rdr); date = getCharacters(rdr);
} }
} }
if (e.isEndElement()) { if (e.isEndElement()) {
if (hasEndTagName(e, "AGENCY_STATUS")) { if (hasEndTagName(e, "AGENCY_STATUS")) {
//System.out.println("Finished processing status: Status = " + status + "; of = " + of + "; date = " + date); //System.out.println("Finished processing status: Status = " + status + "; of = " + of + "; date = " + date);
long statusID, agencyID; long statusID, agencyID;
statusID = getStatus(status); statusID = getStatus(status);
Map<String, Object> agencyProperties = new HashMap<String, Object>(); Map<String, Object> agencyProperties = new HashMap<String, Object>();
agencyProperties.put("agency_no",of); agencyProperties.put("agency_no",of);
agencyID = getAgency(agencyProperties); agencyID = getAgency(agencyProperties);
Map<String, Object> relProperties = new HashMap<String, Object>(); Map<String, Object> relProperties = new HashMap<String, Object>();
relProperties.put("date", date); relProperties.put("date", fixDate(date));
inserter.createRelationship(agencyID, statusID, inserter.createRelationship(agencyID, statusID,
DynamicRelationshipType.withName("HAS_STATUS"), relProperties); DynamicRelationshipType.withName("HAS_STATUS"), relProperties);
   
break; break;
} }
} }
} }
} }
   
private void processAgencyFunction(XMLEventReader rdr) throws Exception { private void processAgencyFunction(XMLEventReader rdr) throws Exception {
String agency = null; String agency = null;
String thesaurus_term = null; String thesaurus_term = null;
String start_date = null; String start_date = null;
String start_date_qual = null; String start_date_qual = null;
String end_date = null; String end_date = null;
String end_date_qual = null; String end_date_qual = null;
   
while (rdr.hasNext()) { while (rdr.hasNext()) {
XMLEvent e = rdr.nextEvent(); XMLEvent e = rdr.nextEvent();
if (e.isStartElement()) { if (e.isStartElement()) {
if (hasStartTagName(e, "FUNCTION_AGENCY_NO")) { if (hasStartTagName(e, "FUNCTION_AGENCY_NO")) {
agency = getCharacters(rdr); agency = getCharacters(rdr);
} else if (hasStartTagName(e, "THESAURUS_TERM")) { } else if (hasStartTagName(e, "THESAURUS_TERM")) {
thesaurus_term = getCharacters(rdr); thesaurus_term = getCharacters(rdr);
} else if (hasStartTagName(e, "START_DATE")) { } else if (hasStartTagName(e, "START_DATE")) {
start_date = getCharacters(rdr); start_date = getCharacters(rdr);
}else if (hasStartTagName(e, "START_DATE_QUAL")) { }else if (hasStartTagName(e, "START_DATE_QUAL")) {
start_date_qual = getCharacters(rdr); start_date_qual = getCharacters(rdr);
}else if (hasStartTagName(e, "END_DATE")) { }else if (hasStartTagName(e, "END_DATE")) {
end_date = getCharacters(rdr); end_date = getCharacters(rdr);
}else if (hasStartTagName(e, "END_DATE_QUAL")) { }else if (hasStartTagName(e, "END_DATE_QUAL")) {
end_date_qual = getCharacters(rdr); end_date_qual = getCharacters(rdr);
} }
} }
if (e.isEndElement()) { if (e.isEndElement()) {
if (hasEndTagName(e, "AGENCY_FUNCTION")) { if (hasEndTagName(e, "AGENCY_FUNCTION")) {
//System.out.println("Finished processing function: Name = " + name + "; of = " + of + "; date = " + date); //System.out.println("Finished processing function: Name = " + name + "; of = " + of + "; date = " + date);
long functionID, agencyID; long functionID, agencyID;
functionID = getFunction(thesaurus_term); functionID = getFunction(thesaurus_term);
Map<String, Object> agencyProperties = new HashMap<String, Object>(); Map<String, Object> agencyProperties = new HashMap<String, Object>();
agencyProperties.put("agency_no",agency); agencyProperties.put("agency_no",agency);
agencyID = getAgency(agencyProperties); agencyID = getAgency(agencyProperties);
Map<String, Object> relProperties = new HashMap<String, Object>(); Map<String, Object> relProperties = new HashMap<String, Object>();
relProperties.put("start_date", start_date); relProperties.put("start_date", Integer.parseInt(start_date));
if (start_date_qual != null && !start_date_qual.equals("(null)")) { if (start_date_qual != null && !start_date_qual.equals("(null)")) {
relProperties.put("start_date_qual", start_date_qual); relProperties.put("start_date_qual", start_date_qual);
} }
if (end_date != null && !end_date.equals("(null)")) { if (end_date != null && !end_date.equals("(null)")) {
relProperties.put("end_date", end_date); relProperties.put("end_date", Integer.parseInt(end_date));
} }
if (end_date_qual != null && !end_date_qual.equals("(null)")) { if (end_date_qual != null && !end_date_qual.equals("(null)")) {
relProperties.put("end_date_qual", end_date_qual); relProperties.put("end_date_qual", end_date_qual);
} }
inserter.createRelationship(agencyID, functionID, inserter.createRelationship(agencyID, functionID,
DynamicRelationshipType.withName("HAS_FUNCTION"), relProperties); DynamicRelationshipType.withName("HAS_FUNCTION"), relProperties);
   
break; break;
} }
} }
} }
} }
  private int fixDate(String date) {
  String[] parts = date.split("-");
  if (parts.length == 3) {
  return Integer.parseInt(""+parts[2]+parts[1]+parts[0]);
  } else {
  return 0;
  }
  }
private String getCharacters(XMLEventReader rdr) throws XMLStreamException { private String getCharacters(XMLEventReader rdr) throws XMLStreamException {
XMLEvent e = rdr.nextEvent(); XMLEvent e = rdr.nextEvent();
if (e.isCharacters()) { if (e.isCharacters()) {
return e.asCharacters().getData(); return e.asCharacters().getData();
} else { } else {
return null; return null;
} }
} }
   
private boolean hasStartTagName(XMLEvent e, String name) { private boolean hasStartTagName(XMLEvent e, String name) {
return e.asStartElement().getName().getLocalPart().toLowerCase().equals(name.toLowerCase()); return e.asStartElement().getName().getLocalPart().toLowerCase().equals(name.toLowerCase());
} }
   
private String getStartTagName(XMLEvent e) {  
return e.asStartElement().getName().getLocalPart();  
}  
   
private boolean hasEndTagName(XMLEvent e, String name) { private boolean hasEndTagName(XMLEvent e, String name) {
return e.asEndElement().getName().getLocalPart().toLowerCase().equals(name.toLowerCase()); return e.asEndElement().getName().getLocalPart().toLowerCase().equals(name.toLowerCase());
} }
   
public void setFilename(String filename) { public void setFilename(String filename) {
this.filename = filename; this.filename = filename;
} }
   
   
} }
   
   
file:b/admin/time.php (new)
  <?php
 
 
  // https://github.com/jadell/neo4jphp
  spl_autoload_register(function ($className) {
  $libPath = '../lib/neo4jphp/lib/';
  $classFile = str_replace('\\',DIRECTORY_SEPARATOR,$className).'.php';
  $classPath = $libPath.$classFile;
  if (file_exists($classPath)) {
  require($classPath);
  }
  });
 
  $nodes = "";
  $edges = "";
  $description = "";
  $nodeList = Array();
 
  function add_node($node) {
  global $nodes,$nodeList;
  if (!in_array($node->getId(),$nodeList)) {
  $start = $node->getProperty("start_date") . $node->getProperty("date");
  $end = $node->getProperty("end_date");
  $nodes.= "<node id='".urlencode($node->getId())."'". ' label="'.htmlentities($node->getProperty("name")).'" '
  . ($start != ""? 'start="'.$start.'" ' : ""). ($end != ""? 'end="'.$end.'" ' : "").'>'
  ."<viz:color b='".rand(0,255)."' g='".rand(0,255)."' r='".rand(0,255)."'/>"
  ."</node>". PHP_EOL;
  $nodeList[] = $node->getId();
  }
  }
 
  function add_edge($rel) {
  global $edges;
  $start = $rel->getProperty("start_date");
  $end = $rel->getProperty("end_date");
  $edges.= "<edge id='".urlencode($rel->getId())."' source='".urlencode($rel->getStartNode()->getId())."' target='".urlencode($rel->getEndNode()->getId())."' ".($start != ""? 'start="'.$start.'" ' : ""). ($end != ""? 'end="'.$end.'" ' : "")."/>". PHP_EOL;
 
  }
 
  function expandNode($node) {
  //global $description;
  //$description .= ($description == ""? "" : " and ").$node->getProperty("name");
  add_node($node);
 
  foreach ($node->getProperties() as $key => $value) {
  // echo "$key: $value\n";
  }
  foreach ($node->getRelationships() as $rel) {
  //echo($rel->getStartNode()->getId()." -> ".$rel->getEndNode()->getId()."<br>");
  add_edge($rel);
  add_node($rel->getStartNode());
  add_node($rel->getEndNode());
  }
  }
 
  //$ids = (isset($_REQUEST['ids']) ? $_REQUEST['ids'] : "");
 
  // Connecting to the default port 7474 on localhost
  $client = new Everyman\Neo4j\Client();
  //$client = new Everyman\Neo4j\Client('192.168.1.127');
  //print_r($client->getServerInfo());
 
  //https://github.com/jadell/neo4jphp/wiki/Caching
  $plugin = new Everyman\Neo4j\Cache\Variable();
  $client->getEntityCache()->setCache($plugin);
 
  /*$memcached = new Memcached();
  $memcached->addServer('localhost', 11211);
 
  $plugin = new Everyman\Neo4j\Cache\Memcached($memcached);
  $client->getEntityCache()->setCache($plugin);*/
 
  /*$requests = explode(";",$ids);
 
 
  foreach ($requests as $request) {
 
  // Array("type" => "path", from=>"1234", to=>"4321","options" => Array())
 
  $parts = explode("-",$request);
 
  $requestType = $parts[0];
  $requestId = $parts[1];
  if ($requestType == 'node') {
  expandNode($client->getNode($requestId));
  } else {
  findNode($requestType,$requestId);
  }
  }
 
  function findNode($type,$id) {
  global $client;
  $typeMapping = Array (
  "agency" => Array("label" => "Agency", "id" => "agencyID")
  );
 
  $queryString =
  "MATCH (n:".$typeMapping[$type]["label"].")".
  "WHERE n.".$typeMapping[$type]["id"]." = {nodeId}".
  "RETURN n";
 
  $query = new Everyman\Neo4j\Cypher\Query($client, $queryString, array('nodeId' => $id));
  $result = $query->getResultSet();
 
  foreach ($result as $row) {
  expandNode( $row[0]);
 
  }
 
  } */
 
  // https://github.com/jadell/neo4jphp/wiki/Paths
 
  /* https://github.com/jadell/neo4jphp/wiki/Traversals
  * $traversal = new Everyman\Neo4j\Traversal($client);
  $traversal->addRelationship('KNOWS', Relationship::DirectionOut)
  ->setPruneEvaluator(Traversal::PruneNone)
  ->setReturnFilter(Traversal::ReturnAll)
  ->setMaxDepth(4);
 
  $nodes = $traversal->getResults($startNode, Traversal::ReturnTypeNode);
 
  */
 
  /*https://github.com/jadell/neo4jphp/wiki/Cypher-and-gremlin-queries
  $queryString = "START n=node({nodeId}) ".
  "MATCH (n)<-[:KNOWS]-(x)".
  "WHERE x.name = {name}".
  "RETURN x";
  $query = new Everyman\Neo4j\Cypher\Query($client, $queryString, array('nodeId' => 1, 'name' => 'Bob'));
  $result = $query->getResultSet();
  foreach ($result as $row) {
  echo $row['x']->getProperty('name') . "\n";
  }*/
 
  $queryString =
  "MATCH (n) WHERE has(n.start_date) and n.start_date < 18870000 RETURN n";
  $description = '1887';
  $query = new Everyman\Neo4j\Cypher\Query($client, $queryString);
  $result = $query->getResultSet();
 
  foreach ($result as $row) {
  expandNode( $row[0]);
 
  }
 
  if (!isset($_REQUEST['debug'])) {
  header('Content-Type: application/gexf+xml');
  header('Content-Disposition: attachment; filename="'.urlencode(str_replace(" ","_",strtolower($description))).'.gexf"');
  }
  echo '<?xml version="1.0" encoding="UTF-8"?>
  <gexf xmlns="http://www.gexf.net/1.2draft" xmlns:viz="http://www.gexf.net/1.2draft/viz" version="1.2">
  <meta lastmodifieddate="2009-03-20">
  <creator>lobbyist.disclosurelo.gs</creator>
  <description>'. $description. '</description>
  </meta>
  <graph mode="dynamic" defaultedgetype="directed" timeformat="date">
 
  <nodes>'. $nodes. '</nodes>
  <edges>'. $edges.' </edges>
  </graph>
  </gexf>'. PHP_EOL;
 
  ?>
file:b/admin/timeline.php (new)
  <?php
  // START n=node(*) OPTIONAL MATCH n<-[r]-parent WITH n, count(parent) AS parents WHERE parents = 0 RETURN n, parents;
  // no parents
 
  // START n=node(*) OPTIONAL MATCH (n:Agency)-[r]->(child:Agency) WITH n, count(child) AS children WHERE children = 0 AND n:Agency RETURN n, children;
  // no children
 
  // MATCH (n) WHERE has(n.start_date) and n.start_date < 18870000 RETURN n
  // time slice
  if (isset($_REQUEST['json'])) {
 
  /* echo '{
  "timeline":
  {
  "headline":"The Main Timeline Headline Goes here",
  "type":"default",
  "text":"<p>Intro body text goes here, some HTML is ok</p>",
  "asset": {
  "media":"http://yourdomain_or_socialmedialink_goes_here.jpg",
  "credit":"Credit Name Goes Here",
  "caption":"Caption text goes here"
  },
  "date": [
  {
  "startDate":"2011,12,10",
  "endDate":"2011,12,11",
  "headline":"Headline Goes Here",
  "text":"<p>Body text goes here, some HTML is OK</p>",
  "tag":"This is Optional",
  "classname":"optionaluniqueclassnamecanbeaddedhere",
  "asset": {
  "media":"http://twitter.com/ArjunaSoriano/status/164181156147900416",
  "thumbnail":"optional-32x32px.jpg",
  "credit":"Credit Name Goes Here",
  "caption":"Caption text goes here"
  }
  }
  ],
  "era": [
  {
  "startDate":"2011,12,10",
  "endDate":"2011,12,11",
  "headline":"Headline Goes Here",
  "text":"<p>Body text goes here, some HTML is OK</p>",
  "tag":"This is Optional"
  }
 
  ]
  }
  }'; */
 
  // https://github.com/jadell/neo4jphp
  spl_autoload_register(function ($className) {
  $libPath = '../lib/neo4jphp/lib/';
  $classFile = str_replace('\\',DIRECTORY_SEPARATOR,$className).'.php';
  $classPath = $libPath.$classFile;
  if (file_exists($classPath)) {
  require($classPath);
  }
  });
 
 
  // Connecting to the default port 7474 on localhost
  $client = new Everyman\Neo4j\Client();
  //$client = new Everyman\Neo4j\Client('192.168.1.127');
  //print_r($client->getServerInfo());
 
 
 
 
  //https://github.com/jadell/neo4jphp/wiki/Cypher-and-gremlin-queries
  $queryString = "MATCH (n) where has(n.agency_no) RETURN n LIMIT 1000";
  $query = new Everyman\Neo4j\Cypher\Query($client, $queryString);
  $dates = Array();
  $result = $query->getResultSet();
  foreach ($result as $row) {
  //$dates[] = Array("startDate" =>"2011,12,10", "endDate"=>"2011,12,11", "headline"=>"Headline Goes Here", "text"=>"<p>Body text goes here, some HTML is OK</p>");
  $name = $row['x']->getProperty('name');
  $startDate = $row['x']->getProperty('start_date');
  $startDate = substr_replace($startDate, ",", 6, 0);
  $startDate = substr_replace($startDate, ",", 4, 0);
  $endDate = $row['x']->getProperty('end_date');
  if ($endDate == " ") $endDate = "20140101";
  $endDate = substr_replace($endDate, ",", 6, 0);
  $endDate = substr_replace($endDate, ",", 4, 0);
 
  $dates[] = Array("startDate" =>$startDate, "endDate"=>$endDate, "headline"=>$name, "text"=>"<p>Body text goes here, some HTML is OK</p>");
 
  }
 
 
 
  $timeline = Array( "headline"=>"The Main Timeline Headline Goes here",
  "type"=>"default",
  "text"=>"<p>Intro body text goes here, some HTML is ok</p>","date"=>$dates);
  echo json_encode(Array("timeline" => $timeline));
  die();
  }
  ?>
 
  <!DOCTYPE html>
  <html lang="en"><!--
 
  88888888888 d8b 888 d8b 888888 d8888b
  888 Y8P 888 Y8P 88b d88P Y88b
  888 888 888 Y88b
  888 888 88888b d88b d88b 888 888 88888b d88b 888 Y888b
  888 888 888 888 88b d8P Y8b 888 888 888 88b d8P Y8b 888 Y88b
  888 888 888 888 888 88888888 888 888 888 888 88888888 888 888
  888 888 888 888 888 Y8b 888 888 888 888 Y8b 88P Y88b d88P
  888 888 888 888 888 Y8888 888 888 888 888 Y8888 888 Y8888P
  d88P
  d88P
  888P
  -->
  <head>
  <title>Revolutionary User Interfaces</title>
  <meta name="description" content="The human computer interface helps to define computing at any one time.">
  <meta charset="utf-8">
  <meta name="apple-mobile-web-app-capable" content="yes">
  <meta name="apple-touch-fullscreen" content="yes">
  <!-- Style-->
  <style>
  html, body {
  height:100%;
  padding: 0px;
  margin: 0px;
  }
  </style>
  <link rel="stylesheet" type="text/css" href="http://cdn.knightlab.com/libs/timeline/latest/css/timeline.css">
  <!-- HTML5 shim, for IE6-8 support of HTML elements--><!--[if lt IE 9]>
  <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script><![endif]-->
  </head>
  <body>
  <!-- BEGIN Timeline Embed -->
  <div id="timeline-embed"></div>
  <script type="text/javascript">
  var timeline_config = {
  width: "100%",
  height: "100%",
  source: 'timeline.php?json=yes',
  font: 'SansitaOne-Kameron'
  }
  </script>
  <script type="text/javascript" src="http://cdn.knightlab.com/libs/timeline/latest/js/storyjs-embed.js"></script>
  <!-- END Timeline Embed-->
 
  </body>
  </html>
Unnamed repository; edit this file to name it for gitweb.
GitPHP by Chris Han