From: Maxious <maxious@lambdacomplex.org>
Date: Sun, 21 Oct 2012 07:22:59 +0000
Subject: Merge branch 'master' of ssh://apples.lambdacomplex.org/git/disclosr
X-Git-Url: http://maxious.lambdacomplex.org/git/?p=disclosr.git&a=commitdiff&h=bc0a17993cf9bd4b882ce56f5ed0a53f9e884bf5
---
Merge branch 'master' of ssh://apples.lambdacomplex.org/git/disclosr


Former-commit-id: 1452e1c2ed1dcc418110e68b2ba9b2cb41036053
---


--- /dev/null
+++ b/admin/importRTKbodies.php
@@ -1,1 +1,56 @@
+<?php
 
+require_once '../include/common.inc.php';
+
+$db = $server->get_db('disclosr-agencies');
+$rows = $db->get_view("app", "byName")->rows;
+$nametoid = Array();
+$accounts = Array();
+foreach ($rows as $row) {
+    $nametoid[trim($row->key)] = $row->value;
+}
+
+function extractCSVAccounts($url, $nameField, $accountField, $filter) {
+    global $accounts, $nametoid;
+    $request = Requests::get($url);
+    echo $url;
+    $Data = str_getcsv($request->body, "\n"); //parse the rows 
+    $headers = Array();
+    foreach ($Data as $num => $line) {
+        $Row = str_getcsv($line, ",");
+        if ($num == 0) {
+            $headers = $Row;
+            print_r($headers);
+        } else {
+            if (isset($Row[array_search($nameField, $headers)])) {
+                $agencyName = $Row[array_search($nameField, $headers)];
+                    if (!in_array(trim($agencyName), array_keys($nametoid))) {
+                        echo "$agencyName missing" . PHP_EOL;
+                    } else {
+                        echo $Row[array_search($nameField, $headers)] . PHP_EOL;
+                             $accounts[$nametoid[trim($agencyName)]]["rtkURLs"][$agencyName] = 'http://www.righttoknow.org.au/body/'.$Row[array_search($accountField, $headers)];
+                    }
+               } else {
+                echo "error finding any agency" . $line . PHP_EOL;
+            }
+        }
+    }
+}
+
+extractCSVAccounts("http://www.righttoknow.org.au/body/all-authorities.csv","Agency","URL name");
+print_r($accounts);
+/* foreach ($accounts as $id => $accountTypes) {
+    echo $id . "<br>" . PHP_EOL;
+    $doc = object_to_array($db->get($id));
+    // print_r($doc);
+
+    foreach ($accountTypes as $accountType => $accounts) {
+        if (!isset($doc["has" . $accountType]) || !is_array($doc["has" . $accountType])) {
+            $doc["has" . $accountType] = Array();
+        }
+        $doc["has" . $accountType] = array_unique(array_merge($doc["has" . $accountType], $accounts));
+    }
+    $db->save($doc);
+}*/
+?>
+

--- /dev/null
+++ b/documents/google676a414ad086cefb.html
@@ -1,1 +1,2 @@
+google-site-verification: google676a414ad086cefb.html
 

--- /dev/null
+++ b/documents/index.php
@@ -1,1 +1,15 @@
+australian disclosure logs
 
+are you looking for more information about:
+contracts
+gov orgs
+lobbyists
+
+1/1/11 title (Dept dfggdfgdf)
+description:
+source link:
+documents:
+    #1 title link
+
+
+rss feed here

--- /dev/null
+++ b/documents/rss.xml.php
@@ -1,1 +1,30 @@
+<?php
 
+// Agency X updated Y,  new files, diff of plain text/link text,
+// feed for just one agency or all
+// This is a minimum example of using the Universal Feed Generator Class
+include("lib/FeedWriter.php");
+//Creating an instance of FeedWriter class.
+$TestFeed = new FeedWriter(RSS2);
+//Setting the channel elements
+//Use wrapper functions for common channelelements
+$TestFeed->setTitle('Last Modified - All');
+$TestFeed->setLink('http://disclosr.lambdacomplex.org/rss.xml.php');
+$TestFeed->setDescription('This is test of creating a RSS 2.0 feed Universal Feed Writer');
+//Retriving informations from database
+$rows = $db->get_view("app", "byLastModified")->rows;
+//print_r($rows);
+foreach ($rows as $row) {
+    //Create an empty FeedItem
+    $newItem = $TestFeed->createNewItem();
+    //Add elements to the feed item
+    $newItem->setTitle($row['name']);
+    $newItem->setLink($row['id']);
+    $newItem->setDate(date("c", $row['metadata']['lastModified']));
+    $newItem->setDescription($row['name']);
+    //Now add the feed item
+    $TestFeed->addItem($newItem);
+}
+//OK. Everything is done. Now genarate the feed.
+$TestFeed->genarateFeed();
+?>

--- /dev/null
+++ b/documents/runScrapers.php

--- /dev/null
+++ b/documents/scrapers/3cd40b1240e987cbcd3f0e67054ce259.py
@@ -1,1 +1,1 @@
-
+http://www.apvma.gov.au/rss/disclosure.rss

--- /dev/null
+++ b/documents/scrapers/8c9421f852c441910bf1d93a57b31d64.py
@@ -1,1 +1,1 @@
-
+http://www.daff.gov.au/about/foi/ips/disclosure_log

--- /dev/null
+++ b/documents/scrapers/be9996f0ac58f71f23d074e82d44ead3.py
@@ -1,1 +1,2 @@
+http://foi.deewr.gov.au/disclosure-log/rss
 

--- /dev/null
+++ b/documents/scrapers/e2a845e55bc9986e6c75c5ad2c508b8d.py
@@ -1,1 +1,1 @@
-
+www.finance.gov.au/foi/disclosure-log/foi-rss.xml

--- /dev/null
+++ b/documents/scrapers/rtk.py
@@ -1,1 +1,1 @@
-
+http://www.righttoknow.org.au/feed/search/%20(latest_status:successful%20OR%20latest_status:partially_successful)

--- /dev/null
+++ b/documents/search.php
@@ -1,1 +1,25 @@
+<?php
+include_once('include/common.inc.php');
+include_header('Search');
+?>
+<div class="foundation-header">
+    <h1><a href="search.php">Search</a></h1>
+</div>
+<form>
+    <input type="text" name="q" value="<?php if (isset($_REQUEST['q']))echo $_REQUEST['q'];?>"/>
+    <input type="submit"/>
+</form>
 
+<?php
+if (isset($_REQUEST['q'])) {
+    $request = Requests::get($serverAddr."disclosr-documents/_fti/_design/lucene/by_all?include_docs=true&q=".$_REQUEST['q']);
+    $results = json_decode($request->body);
+    $db = $server->get_db('disclosr-documents');
+    foreach ($results->rows as $result) {
+        //print_r($result);
+         //$row = $db->get($result->id);
+        echo $result->doc->_id." ".$result->doc->url."<br>".PHP_EOL;
+    }
+}
+include_footer();
+?>

--- a/rss.xml.php
+++ /dev/null
@@ -1,30 +1,1 @@
-<?php
 
-// Agency X updated Y,  new files, diff of plain text/link text,
-// feed for just one agency or all
-// This is a minimum example of using the Universal Feed Generator Class
-include("lib/FeedWriter.php");
-//Creating an instance of FeedWriter class.
-$TestFeed = new FeedWriter(RSS2);
-//Setting the channel elements
-//Use wrapper functions for common channelelements
-$TestFeed->setTitle('Last Modified - All');
-$TestFeed->setLink('http://disclosr.lambdacomplex.org/rss.xml.php');
-$TestFeed->setDescription('This is test of creating a RSS 2.0 feed Universal Feed Writer');
-//Retriving informations from database
-$rows = $db->get_view("app", "byLastModified")->rows;
-//print_r($rows);
-foreach ($rows as $row) {
-    //Create an empty FeedItem
-    $newItem = $TestFeed->createNewItem();
-    //Add elements to the feed item
-    $newItem->setTitle($row['name']);
-    $newItem->setLink($row['id']);
-    $newItem->setDate(date("c", $row['metadata']['lastModified']));
-    $newItem->setDescription($row['name']);
-    //Now add the feed item
-    $TestFeed->addItem($newItem);
-}
-//OK. Everything is done. Now genarate the feed.
-$TestFeed->genarateFeed();
-?>

--- a/search.php
+++ /dev/null
@@ -1,25 +1,1 @@
-<?php
-include_once('include/common.inc.php');
-include_header('Search');
-?>
-<div class="foundation-header">
-    <h1><a href="search.php">Search</a></h1>
-</div>
-<form>
-    <input type="text" name="q" value="<?php if (isset($_REQUEST['q']))echo $_REQUEST['q'];?>"/>
-    <input type="submit"/>
-</form>
 
-<?php
-if (isset($_REQUEST['q'])) {
-    $request = Requests::get($serverAddr."disclosr-documents/_fti/_design/lucene/by_all?include_docs=true&q=".$_REQUEST['q']);
-    $results = json_decode($request->body);
-    $db = $server->get_db('disclosr-documents');
-    foreach ($results->rows as $result) {
-        //print_r($result);
-         //$row = $db->get($result->id);
-        echo $result->doc->_id." ".$result->doc->url."<br>".PHP_EOL;
-    }
-}
-include_footer();
-?>