beginnings of document scrapers mark 2
[disclosr.git] / admin / importAustraliaGovAuGov2.php
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
<?php
 
require_once '../include/common.inc.php';
 
$db = $server->get_db('disclosr-agencies');
$rows = $db->get_view("app", "byName")->rows;
$nametoid = Array();
$accounts = Array();
foreach ($rows as $row) {
    $nametoid[trim($row->key)] = $row->value;
}
 
function extractCSVAccounts($url, $nameField, $accountField, $filter) {
    global $accounts, $nametoid;
    $request = Requests::get($url);
    $Data = str_getcsv($request->body, "\n"); //parse the rows 
    $headers = Array();
    foreach ($Data as $num => $line) {
        $Row = str_getcsv($line, ",");
        if ($num == 0) {
            
        } else if ($num == 1) {
            $headers = $Row;
            //print_r($headers);
        } else {
            if (isset($Row[array_search($nameField, $headers)])) {
                $agencyName = $Row[array_search($nameField, $headers)];
                if (!$filter || $Row[array_search("State", $headers)] == "NAT") {
                    if (!in_array(trim($agencyName), array_keys($nametoid))) {
                        echo "$agencyName missing" . PHP_EOL;
                    } else {
                        // echo $Row[array_search($nameField, $headers)] . PHP_EOL;
                    }
                }
            } else {
                //echo "error finding agency" . $line . PHP_EOL;
            }
        }
    }
}
 
// http://agimo.govspace.gov.au/page/gov2register/
// twitter
//extractCSVAccounts("https://docs.google.com/spreadsheet/pub?key=0Ap1exl80wB8OdHNKVmQ5RVlvQWpibDAxNHkzcU1nV2c&single=true&gid=0&output=csv", "Agency/Body/Event", "", true);
// RSS
// https://docs.google.com/spreadsheet/pub?hl=en_GB&hl=en_GB&key=0Ah41IAK0HzSTdGJxandJREhLSGlWWUZfZ2xKOTNHZ0E&output=csv
// facebook 
extractCSVAccounts("https://docs.google.com/spreadsheet/pub?hl=en_GB&hl=en_GB&key=0Ah41IAK0HzSTdGtjcW9vOXdyZ3pOV21vQU51VmhzQnc&single=true&gid=0&output=csv","Agency","Name");
 
/*
 * http://australia.gov.au/news-and-media/media-release-rss-feeds
 * http://australia.gov.au/news-and-media/social-media/blogs
 * http://australia.gov.au/news-and-media/social-media/twitter
 * http://australia.gov.au/news-and-media/social-media/facebook
 * http://australia.gov.au/news-and-media/social-media/youtube
 * http://australia.gov.au/news-and-media/social-media/flickr
 * http://australia.gov.au/news-and-media/social-media/apps http://www.harmony.gov.au/get-involved/app-downloads.htm http://www.em.gov.au/Resources/Pages/Before-the-Storm-phone-game.aspx
 * http://australia.gov.au/news-and-media/social-media/podcasts
 */
?>