add other metadata for datagov export into extras field
[disclosr.git] / documents / scrapers / 0049d35216493c545ef5f7f000e6b252.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
import sys
import os
sys.path.insert(0, os.path.join(os.path.dirname(__file__) or '.', '../'))
import genericScrapers
import traceback
try:
        import amonpy
        amonpy.config.address = 'http://amon_instance:port'
        amonpy.config.secret_key = 'the secret key from /etc/amon.conf'
        amon_available = True
except ImportError:
        amon_available = False
 
class ScraperImplementation(genericScrapers.GenericPDFDisclogScraper):
 
    def __init__(self):
        super(ScraperImplementation, self).__init__()
 
 
if __name__ == '__main__':
    print 'Subclass:', issubclass(ScraperImplementation,
         genericScrapers.GenericPDFDisclogScraper)
    print 'Instance:', isinstance(ScraperImplementation(),
         genericScrapers.GenericPDFDisclogScraper)
    try:
        ScraperImplementation().doScrape()
    except Exception, err:
        sys.stderr.write('ERROR: %s\n' % str(err))
        print ‘Error Reason: ‘, err.__doc__
        printException: ‘, err.__class__
        print traceback.format_exc()
        if amon_available:
               data = {
                        'exception_class': '',
                        'url': '',
                        'backtrace': ['exception line ', 'another exception line'],
                        'enviroment': '',
                        
                        # In 'data' you can add request information, session variables - it's a recursive 
                        # dictionary, so you can literally add everything important for your specific case
                        # The dictionary doesn't have a specified structure, the keys below are only example
                        'data': {'request': '', 'session': '', 'more': ''}
 
                }
 
                amonpy.exception(data)
        pass