| <?php | <?php |
| include_once("../include/common.inc.php"); | include_once("../include/common.inc.php"); |
| require($basePath . 'lib/phpquery/phpQuery/phpQuery.php'); | require($basePath . 'lib/phpquery/phpQuery/phpQuery.php'); |
| setlocale(LC_CTYPE, 'C'); | setlocale(LC_CTYPE, 'C'); |
| $db = $server->get_db('disclosr-agencies'); | $db = $server->get_db('disclosr-agencies'); |
| // metatags | |
| try { | |
| $agencies = $db->get_view("app", "byCanonicalName", null, true)->rows; | |
| //print_r($rows); | |
| foreach ($agencies as $agency) { | |
| if (isset($agency->value->scrapeDepth)) { | |
| unset($agency->value->scrapeDepth); | |
| } | |
| if (isset($agency->value->lastScraped)) { | |
| unset($agency->value->lastScraped); | |
| } | |
| $db->save($agency->value); | |
| echo "<hr>"; | |
| flush(); | |
| } | |
| } catch (SetteeRestClientException $e) { | |
| setteErrorHandler($e); | |
| } | |
| // metatags | |
| try { | try { |
| $agencies = $db->get_view("app", "byCanonicalName", null, true)->rows; | $agencies = $db->get_view("app", "byCanonicalName", null, true)->rows; |
| //print_r($rows); | //print_r($rows); |
| foreach ($agencies as $agency) { | foreach ($agencies as $agency) { |
| //echo $agency->value->name . " ".$agency->value->website."<br />\n"; | //echo $agency->value->name . " ".$agency->value->website."<br />\n"; |
| // print_r($agency); | // print_r($agency); |
| //hasRestricitiveLicence" hasRestrictiveLicense -> has Restrictive Licence | //hasRestricitiveLicence" hasRestrictiveLicense -> has Restrictive Licence |
| // "hasYoutube" -> Tube | // "hasYoutube" -> Tube |
| // "comment" -> "comments" | // "comment" -> "comments" |
| if (!isset($agency->value->metaTags) && isset($agency->value->website)) { | if (!isset($agency->value->metaTags) && isset($agency->value->website)) { |
| echo $agency->value->name . " ".$agency->value->website."<br />\n"; | echo $agency->value->name . " " . $agency->value->website . "<br />\n"; |
| $agency->value->metaTags = Array(); | $agency->value->metaTags = Array(); |
| $request = Requests::get($agency->value->website); | $request = Requests::get($agency->value->website); |
| $html = phpQuery::newDocumentHTML($request->body); | $html = phpQuery::newDocumentHTML($request->body); |
| phpQuery::selectDocument($html); | phpQuery::selectDocument($html); |
| foreach (pq('meta')->elements as $meta) { | foreach (pq('meta')->elements as $meta) { |
| $tagName = $meta->getAttribute('name');; | $tagName = $meta->getAttribute('name'); |
| ; | |
| $content = $meta->getAttribute('content'); | $content = $meta->getAttribute('content'); |
| if ($tagName != "") { | if ($tagName != "") { |
| echo "$tagName == $content <br>\n"; | echo "$tagName == $content <br>\n"; |
| $agency->value->metaTags[$tagName] = $content; | $agency->value->metaTags[$tagName] = $content; |
| } | } |
| } | } |
| //print_r($agency->value->metaTags); | //print_r($agency->value->metaTags); |
| $db->save($agency->value); | $db->save($agency->value); |
| echo "<hr>"; | echo "<hr>"; |
| flush(); | flush(); |
| } | } |
| } | } |
| } catch (SetteeRestClientException $e) { | } catch (SetteeRestClientException $e) { |
| setteErrorHandler($e); | setteErrorHandler($e); |
| } | } |
| ?> | ?> |