From: root Date: Wed, 11 Dec 2013 08:33:49 +0000 Subject: fix by org stats X-Git-Url: https://maxious.lambdacomplex.org/git/?p=ckanext-dga-stats.git&a=commitdiff&h=b644f2e1bcf44b362de1ed3a83ed966b2bf06827 --- fix by org stats --- --- /dev/null +++ b/.gitignore @@ -1,1 +1,36 @@ +*.py[cod] +# C extensions +*.so + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + --- a/README.md +++ b/README.md @@ -5,4 +5,5 @@ * Remove private datasets from all statistics (except top users) * Add summary page * Add activity summary page +* Add organisation public/private dataset count page --- a/ckanext/dga_stats/controller.py +++ b/ckanext/dga_stats/controller.py @@ -16,6 +16,9 @@ c.top_package_owners = stats.top_package_owners() c.summary_stats = stats.summary_stats() c.activity_counts = stats.activity_counts() + c.by_org = stats.by_org() + c.user_access_list = stats.user_access_list() + c.recent_datasets = stats.recent_datasets() c.new_packages_by_week = rev_stats.get_by_week('new_packages') c.deleted_packages_by_week = rev_stats.get_by_week('deleted_packages') c.num_packages_by_week = rev_stats.get_num_packages_by_week() --- a/ckanext/dga_stats/stats.py +++ b/ckanext/dga_stats/stats.py @@ -2,6 +2,7 @@ from pylons import config from sqlalchemy import Table, select, func, and_ +from sqlalchemy.sql.expression import text import ckan.plugins as p import ckan.model as model @@ -42,7 +43,7 @@ package_revision = table('package_revision') package = table('package') s = select([package_revision.c.id, func.count(package_revision.c.revision_id)], from_obj=[package_revision.join(package)]).\ - where(package.c.private == 'f').\ + where(package.c.private == 'f').\ group_by(package_revision.c.id).\ order_by(func.count(package_revision.c.revision_id).desc()).\ limit(limit) @@ -52,16 +53,28 @@ @classmethod def largest_groups(cls, limit=10): - member = table('member') - package = table('package') - s = select([member.c.group_id, func.count(member.c.table_id)]).\ + member = table('member') + s = select([member.c.group_id, func.count(member.c.table_id)]).\ group_by(member.c.group_id).\ - where(and_(member.c.group_id!=None, member.c.table_name=='package', package.c.private == 'f')).\ - order_by(func.count(member.c.table_id).desc()).\ - limit(limit) - - res_ids = model.Session.execute(s).fetchall() - res_groups = [(model.Session.query(model.Group).get(unicode(group_id)), val) for group_id, val in res_ids] + where(member.c.group_id!=None).\ + where(member.c.table_name=='package').\ + where(member.c.capacity=='public').\ + order_by(func.count(member.c.table_id).desc()) + #limit(limit) + + res_ids = model.Session.execute(s).fetchall() + res_groups = [(model.Session.query(model.Group).get(unicode(group_id)), val) for group_id, val in res_ids] + return res_groups + + @classmethod + def by_org(cls, limit=10): + connection = model.Session.connection() + res = connection.execute("select package.owner_org, package.private, count(*) from package \ + inner join \"group\" on package.owner_org = \"group\".id \ + where package.state='active'\ + group by package.owner_org,\"group\".name, package.private \ + order by \"group\".name, package.private;").fetchall(); + res_groups = [(model.Session.query(model.Group).get(unicode(group_id)), private, val) for group_id, private, val in res] return res_groups @classmethod @@ -95,10 +108,9 @@ package_role = table('package_role') user_object_role = table('user_object_role') package = table('package') -# join(package, package_role.c.package_id == package.c.id).\ -# where(package.c.private == 'f').\ - s = select([user_object_role.c.user_id, func.count(user_object_role.c.role)], from_obj=[user_object_role.join(package_role)]).\ + s = select([user_object_role.c.user_id, func.count(user_object_role.c.role)], from_obj=[user_object_role.join(package_role).join(package, package_role.c.package_id == package.c.id)]).\ where(user_object_role.c.role==model.authz.Role.ADMIN).\ + where(package.c.private == 'f').\ where(user_object_role.c.user_id!=None).\ group_by(user_object_role.c.user_id).\ order_by(func.count(user_object_role.c.role).desc()).\ @@ -110,10 +122,10 @@ @classmethod def summary_stats(cls): connection = model.Session.connection() -# select name,role from user_object_role inner join \"user\" on user_object_role.user_id = \"user\".id where name not in ('logged_in','visitor') group by name,role" - + +# select 'Total Archived Datasets', count(*) from package where (state='active' or state='draft' or state='draft-complete') and private = 't' union \ res = connection.execute("SELECT 'Total Organisations', count(*) from \"group\" where type = 'organization' and state = 'active' union \ - select 'Total Datasets', count(*) from package where state='active' or state='draft' or state='draft-complete' union \ + select 'Total Datasets', count(*) from package where (state='active' or state='draft' or state='draft-complete') and private = 'f' union \ select 'Total Data Files/Resources', count(*) from resource where state='active'").fetchall(); return res @@ -123,6 +135,22 @@ connection = model.Session.connection() res = connection.execute("select to_char(timestamp, 'YYYY-MM') as month,activity_type, count(*) from activity group by month, activity_type order by month;").fetchall(); return res + + @classmethod + def user_access_list(cls): + connection = model.Session.connection() + res = connection.execute("select name,sysadmin,role from user_object_role right outer join \"user\" on user_object_role.user_id = \"user\".id where name not in ('logged_in','visitor') group by name,sysadmin,role order by sysadmin desc, role asc;").fetchall(); + return res + + @classmethod + def recent_datasets(cls): + activity = table('activity') + package = table('package') + s = select([func.max(activity.c.timestamp),package.c.id, activity.c.activity_type], from_obj=[activity.join(package,activity.c.object_id == package.c.id)]).where(package.c.private == 'f').\ + where(activity.c.timestamp > func.now() - text("interval '60 day'")).group_by(package.c.id,activity.c.activity_type).order_by(func.max(activity.c.timestamp)) + result = model.Session.execute(s).fetchall() + return [(datetime2date(timestamp), model.Session.query(model.Package).get(unicode(package_id)), activity_type) for timestamp,package_id,activity_type in result] + class RevisionStats(object): @@ -193,7 +221,8 @@ # be 'for all time' else you get first revision in the time period. package_revision = table('package_revision') revision = table('revision') - s = select([package_revision.c.id, func.min(revision.c.timestamp)], from_obj=[package_revision.join(revision)]).\ + package = table('package') + s = select([package_revision.c.id, func.min(revision.c.timestamp)], from_obj=[package_revision.join(revision).join(package)]).\ where(package.c.private == 'f').\ group_by(package_revision.c.id).order_by(func.min(revision.c.timestamp)) res = model.Session.execute(s).fetchall() # [(id, datetime), ...] @@ -221,8 +250,10 @@ # be 'for all time' else you get first revision in the time period. package_revision = table('package_revision') revision = table('revision') - s = select([package_revision.c.id, func.min(revision.c.timestamp)], from_obj=[package_revision.join(revision)]).\ + package = table('package') + s = select([package_revision.c.id, func.min(revision.c.timestamp)], from_obj=[package_revision.join(revision).join(package)]).\ where(package_revision.c.state==model.State.DELETED).\ + where(package.c.private == 'f').\ group_by(package_revision.c.id).\ order_by(func.min(revision.c.timestamp)) res = model.Session.execute(s).fetchall() # [(id, datetime), ...] --- a/ckanext/dga_stats/templates/ckanext/stats/index.html +++ b/ckanext/dga_stats/templates/ckanext/stats/index.html @@ -29,6 +29,7 @@

{{ _('No groups') }}

{% endif %} +{% if h.check_access('sysadmin') %}

{{ _('Site Activity Log') }}

{% if c.activity_counts %} @@ -54,7 +55,57 @@

{{ _('No groups') }}

{% endif %}
- +
+

{{ _('Recent Datasets') }}

+ {% if c.recent_datasets %} + + + + + + + + + + {% for date,package,newmodified in c.recent_datasets %} + + + + + + {% endfor %} + +
{{ _('Date') }}{{ _('Dataset') }}{{ _('New/Modified') }}
{{ date }}{{ h.link_to(package.title or package.name, h.url_for(controller='package', action='read', id=package.name)) }}{{ newmodified }}
+ {% else %} +

{{ _('No groups') }}

+ {% endif %} +
+
+

{{ _('User Access List') }}

+ {% if c.user_access_list %} + + + + + + + + + + {% for username,sysadmin,role in c.user_access_list %} + + + + + + {% endfor %} + +
{{ _('Username') }}{{ _('Sysadmin') }}{{ _('Organisational Role') }}
{{ username }}{{ sysadmin }}{{ role }}
+ {% else %} +

{{ _('No groups') }}

+ {% endif %} +
+{% endif %}

{{ _('Total number of Datasets') }}

@@ -163,7 +214,7 @@ {{ _('Number of datasets') }} - + {% for group, num_packages in c.largest_groups %} {{ h.link_to(group.title or group.name, h.url_for(controller='group', action='read', id=group.name)) }} @@ -173,33 +224,10 @@ {% else %} -

{{ _('No groups') }}

- {% endif %} -
-
-

{{ _('Largest Groups') }}

- {% if c.largest_groups %} - - - - - - - - - {% for group, num_packages in c.largest_groups %} - - - - - {% endfor %} - -
{{ _('Group') }}{{ _('Number of datasets') }}
{{ h.link_to(group.title or group.name, h.url_for(controller='group', action='read', id=group.name)) }}{{ num_packages }}
- {% else %} -

{{ _('No groups') }}

- {% endif %} -
- +

{{ _('No groups') }}

+ {% endif %} + +{% if False %}

{{ _('Top Tags') }}

@@ -219,7 +247,6 @@
-

{{ _('Users Owning Most Datasets') }}

@@ -239,6 +266,38 @@
+{% endif %} +
+

{{ _('Datasets by Organization') }}

+ {% if c.by_org %} + + + + + + + + + + {% for group,private, num_packages in c.by_org %} + {% if private == False or h.check_access('sysadmin') %} + + + {% if private == True %} + + {% else %} + + {% endif %} + + + {% endif %} + {% endfor %} + +
{{ _('Group') }}{{ _('Public/Archived') }}{{ _('Number of datasets') }}
{{ h.link_to(group.title or group.name, h.url_for(controller='organization', action='read', id=group.name)) }}ArchivedPublic{{ num_packages }}
+ {% else %} +

{{ _('No groups') }}

+ {% endif %} +
{% endblock %} @@ -247,15 +306,19 @@

{{ _('Statistics Menu') }}