diff --git a/jsapp/js/api/react-query/audit-logs-superusers.ts b/jsapp/js/api/react-query/server-logs-superusers.ts similarity index 100% rename from jsapp/js/api/react-query/audit-logs-superusers.ts rename to jsapp/js/api/react-query/server-logs-superusers.ts diff --git a/kobo/apps/audit_log/views.py b/kobo/apps/audit_log/views.py index cf5140929c..9f92d979be 100644 --- a/kobo/apps/audit_log/views.py +++ b/kobo/apps/audit_log/views.py @@ -49,7 +49,7 @@ @extend_schema( - tags=['Audit logs (superusers)'], + tags=['Server logs (superusers)'], ) @extend_schema_view( list=extend_schema( @@ -94,7 +94,7 @@ class AuditLogViewSet(mixins.ListModelMixin, viewsets.GenericViewSet): @extend_schema_view( list=extend_schema( - tags=['Audit logs (superusers)'], + tags=['Server logs (superusers)'], description=read_md('audit_log', 'access_logs/list'), responses=open_api_200_ok_response( AccessLogResponse, @@ -157,7 +157,7 @@ class AccessLogViewSet(AuditLogViewSet): require_auth=False, validate_payload=False, ), - tags=['Audit logs (superusers)'], + tags=['Server logs (superusers)'], ) ) class AllProjectHistoryLogViewSet(AuditLogViewSet): @@ -188,7 +188,7 @@ class AllProjectHistoryLogViewSet(AuditLogViewSet): require_auth=False, validate_payload=False, ), - tags=['Audit logs (superusers)'], + tags=['Server logs (superusers)'], ) @extend_schema( methods=['POST'], @@ -201,7 +201,7 @@ class AllProjectHistoryLogViewSet(AuditLogViewSet): require_auth=False, validate_payload=False, ), - tags=['Audit logs (superusers)'], + tags=['Server logs (superusers)'], ) @action(detail=False, methods=['GET', 'POST']) def export(self, request, *args, **kwargs): @@ -469,7 +469,7 @@ def list(self, request, *args, **kwargs): @extend_schema( - tags=['Audit logs (superusers)'], + tags=['Server logs (superusers)'], ) @extend_schema_view( list=extend_schema( diff --git a/kobo/apps/stripe/utils/billing_dates.py b/kobo/apps/stripe/utils/billing_dates.py index 091a2b6676..6e30efc305 100644 --- a/kobo/apps/stripe/utils/billing_dates.py +++ b/kobo/apps/stripe/utils/billing_dates.py @@ -3,6 +3,7 @@ from zoneinfo import ZoneInfo from dateutil.relativedelta import relativedelta +from django.conf import settings from django.db.models import F, Max, Q, Window from django.utils import timezone @@ -10,17 +11,34 @@ from kobo.apps.organizations.types import BillingDates from kobo.apps.stripe.constants import ACTIVE_STRIPE_STATUSES from kobo.apps.stripe.utils.import_management import requires_stripe +from kobo.apps.user_reports.typing_aliases import OrganizationIterator -@requires_stripe def get_current_billing_period_dates_by_org( - orgs: list[Organization] = None, **kwargs + orgs: OrganizationIterator = None, **kwargs ) -> dict[str, BillingDates]: now = timezone.now().replace(tzinfo=ZoneInfo('UTC')) first_of_this_month = datetime(now.year, now.month, 1, tzinfo=ZoneInfo('UTC')) first_of_next_month = first_of_this_month + relativedelta(months=1) + if not settings.STRIPE_ENABLED: + results = {} + if orgs is not None: + for org in orgs: + results[org.id] = { + 'start': first_of_this_month, + 'end': first_of_next_month + } + return results + + for org in Organization.objects.all(): + results[org.id] = { + 'start': first_of_this_month, + 'end': first_of_next_month + } + return results + # check 1: look for active subscriptions all_active_billing_dates = get_current_billing_period_dates_for_active_plans(orgs) diff --git a/kobo/apps/user_reports/__init__.py b/kobo/apps/user_reports/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/kobo/apps/user_reports/apps.py b/kobo/apps/user_reports/apps.py new file mode 100644 index 0000000000..8b08b65c06 --- /dev/null +++ b/kobo/apps/user_reports/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class UserReportsConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'kobo.apps.user_reports' diff --git a/kobo/apps/user_reports/migrations/0001_initial.py b/kobo/apps/user_reports/migrations/0001_initial.py new file mode 100644 index 0000000000..e650e02731 --- /dev/null +++ b/kobo/apps/user_reports/migrations/0001_initial.py @@ -0,0 +1,208 @@ +from django.db import migrations, models +from django.db.models import Q + +import kpi.models.abstract_models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name='BillingAndUsageSnapshotRun', + fields=[ + ( + 'id', + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID' + ) + ), + ('uid', kpi.fields.kpi_uid.KpiUidField(_null=False, uid_prefix='busr')), + ( + 'status', + models.CharField( + choices=[ + ('in_progress', 'In Progress'), + ('completed', 'Completed'), + ('aborted', 'Aborted') + ], + default='in_progress', + max_length=32 + ) + ), + ('last_processed_org_id', models.CharField(blank=True, null=True)), + ('details', models.JSONField(blank=True, null=True)), + ('singleton', models.BooleanField(default=True, editable=False)), + ( + 'date_created', + models.DateTimeField( + default=kpi.models.abstract_models._get_default_datetime + ), + ), + ( + 'date_modified', + models.DateTimeField( + default=kpi.models.abstract_models._get_default_datetime + ), + ), + ], + options={ + 'ordering': ['-date_created'], + }, + ), + migrations.CreateModel( + name='BillingAndUsageSnapshot', + fields=[ + ( + 'id', + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID' + ) + ), + ( + 'effective_user_id', + models.IntegerField(blank=True, null=True, db_index=True) + ), + ( + 'last_snapshot_run', + models.ForeignKey( + on_delete=models.deletion.CASCADE, + related_name='snapshots', + to='user_reports.billingandusagesnapshotrun', + ), + ), + ( + 'organization', + models.OneToOneField( + on_delete=models.deletion.CASCADE, + to='organizations.organization', + ), + ), + ('total_storage_bytes', models.BigIntegerField(default=0)), + ('total_submission_count_all_time', models.BigIntegerField(default=0)), + ( + 'total_submission_count_current_period', + models.BigIntegerField(default=0) + ), + ('billing_period_start', models.DateTimeField(blank=True, null=True)), + ('billing_period_end', models.DateTimeField(blank=True, null=True)), + ( + 'date_created', + models.DateTimeField( + default=kpi.models.abstract_models._get_default_datetime + ), + ), + ( + 'date_modified', + models.DateTimeField( + default=kpi.models.abstract_models._get_default_datetime + ), + ), + ('submission_limit', models.BigIntegerField(blank=True, null=True)), + ('storage_bytes_limit', models.BigIntegerField(blank=True, null=True)), + ('asr_seconds_limit', models.BigIntegerField(blank=True, null=True)), + ('mt_characters_limit', models.BigIntegerField(blank=True, null=True)), + ], + ), + migrations.AddIndex( + model_name='billingandusagesnapshot', + index=models.Index(fields=['effective_user_id'], name='idx_bau_user'), + ), + migrations.AddIndex( + model_name='billingandusagesnapshot', + index=models.Index(fields=['date_created'], name='idx_bau_created'), + ), + # Add index for runs + migrations.AddIndex( + model_name='billingandusagesnapshotrun', + index=models.Index( + fields=['status', 'date_modified'], + name='idx_bau_run_status_expires' + ), + ), + migrations.AddConstraint( + model_name='billingandusagesnapshotrun', + constraint=models.UniqueConstraint( + fields=('singleton',), + condition=Q(status='in_progress'), + name='uniq_run_in_progress' + ), + ), + migrations.AddConstraint( + model_name='billingandusagesnapshot', + constraint=models.UniqueConstraint( + fields=('organization',), + name='uniq_snapshot_per_org' + ), + ), + # Register the materialized-view model state (unmanaged) so Django knows + # the model exists for ORM queries, but doesn't try to create a migration + # file for it. The actual materialized view is created by the subsequent + # migration `0002_create_user_reports_mv.py`. + migrations.SeparateDatabaseAndState( + database_operations=[], + state_operations=[ + migrations.CreateModel( + name='UserReports', + fields=[ + ( + 'id', + models.CharField(max_length=80, primary_key=True) + ), + ( + 'extra_details_uid', + models.CharField(max_length=255, null=True, blank=True) + ), + ('username', models.CharField(max_length=150)), + ('first_name', models.CharField(max_length=150)), + ('last_name', models.CharField(max_length=150)), + ('email', models.CharField(max_length=254)), + ('is_superuser', models.BooleanField()), + ('is_staff', models.BooleanField()), + ('is_active', models.BooleanField()), + ('date_joined', models.CharField(max_length=64)), + ( + 'last_login', + models.CharField(max_length=64, null=True, blank=True) + ), + ('validated_email', models.BooleanField()), + ('validated_password', models.BooleanField()), + ('mfa_is_active', models.BooleanField()), + ('sso_is_active', models.BooleanField()), + ('accepted_tos', models.BooleanField()), + ('social_accounts', models.JSONField(null=True, blank=True)), + ('organization', models.JSONField(null=True, blank=True)), + ('metadata', models.JSONField(null=True, blank=True)), + ('subscriptions', models.JSONField(null=True, blank=True)), + ('asset_count', models.IntegerField(default=0)), + ('deployed_asset_count', models.IntegerField(default=0)), + ( + 'current_period_start', + models.DateTimeField(null=True, blank=True) + ), + ( + 'current_period_end', + models.DateTimeField(null=True, blank=True) + ), + ( + 'service_usage', + models.JSONField(null=True, blank=True), + ), + ], + options={ + 'managed': False, + 'db_table': 'user_reports_userreportsmv', + }, + ), + ], + ), + ] diff --git a/kobo/apps/user_reports/migrations/0002_create_user_reports_mv.py b/kobo/apps/user_reports/migrations/0002_create_user_reports_mv.py new file mode 100644 index 0000000000..2682dd36a8 --- /dev/null +++ b/kobo/apps/user_reports/migrations/0002_create_user_reports_mv.py @@ -0,0 +1,469 @@ +# flake8: noqa: E501 + +from django.conf import settings +from django.db import migrations + + +CREATE_MV_BASE_SQL = f""" + CREATE MATERIALIZED VIEW user_reports_userreportsmv AS + WITH user_nlp_usage AS ( + SELECT + nuc.user_id, + COALESCE(SUM(nuc.total_asr_seconds), 0) AS total_asr_seconds, + COALESCE(SUM(nuc.total_mt_characters), 0) AS total_mt_characters + FROM trackers_nlpusagecounter nuc + GROUP BY nuc.user_id + ), + user_assets AS ( + SELECT + a.owner_id as user_id, + COUNT(*) as total_assets, + COUNT(*) FILTER (WHERE a._deployment_status = 'deployed') as deployed_assets + FROM kpi_asset a + WHERE a.pending_delete = false + GROUP BY a.owner_id + ), + user_role_map AS ( + SELECT + au.id AS user_id, + CASE + WHEN EXISTS ( + SELECT 1 + FROM organizations_organizationowner o + JOIN organizations_organizationuser ou_owner ON ou_owner.id = o.organization_user_id + WHERE ou_owner.user_id = au.id + ) THEN 'owner' + WHEN EXISTS ( + SELECT 1 FROM organizations_organizationuser ou WHERE ou.user_id = au.id AND ou.is_admin IS TRUE + ) THEN 'admin' + WHEN EXISTS ( + SELECT 1 FROM organizations_organizationuser ou WHERE ou.user_id = au.id + ) THEN 'member' + ELSE 'external' + END AS user_role + FROM auth_user au + ), + user_billing_periods AS ( + SELECT DISTINCT + au.id as user_id, + bus.billing_period_start AS current_period_start, + bus.billing_period_end AS current_period_end, + bus.organization_id, + bus.submission_limit, + bus.storage_bytes_limit, + bus.asr_seconds_limit, + bus.mt_characters_limit, + COALESCE(bus.total_storage_bytes, 0) as total_storage_bytes, + COALESCE(bus.total_submission_count_all_time, 0) as total_submission_count_all_time, + COALESCE(bus.total_submission_count_current_period, 0) as total_submission_count_current_period + FROM auth_user au + LEFT JOIN organizations_organizationuser ou ON au.id = ou.user_id + LEFT JOIN user_reports_billingandusagesnapshot bus ON ou.organization_id = bus.organization_id + ), + nlp_period_agg AS ( + SELECT + nuc.user_id, + SUM( + CASE WHEN nuc.date >= ubp.current_period_start AND nuc.date <= ubp.current_period_end + THEN nuc.total_asr_seconds ELSE 0 END + ) AS total_nlp_usage_asr_seconds_current_period, + SUM( + CASE WHEN nuc.date >= ubp.current_period_start AND nuc.date <= ubp.current_period_end + THEN nuc.total_mt_characters ELSE 0 END + ) AS total_nlp_usage_mt_characters_current_period + FROM trackers_nlpusagecounter nuc + JOIN user_billing_periods ubp ON nuc.user_id = ubp.user_id + GROUP BY nuc.user_id + ), + user_current_period_usage AS ( + SELECT + ubp.user_id, + ubp.current_period_start, + ubp.current_period_end, + ubp.organization_id, + COALESCE(na.total_nlp_usage_asr_seconds_current_period, 0) AS total_nlp_usage_asr_seconds_current_period, + COALESCE(na.total_nlp_usage_mt_characters_current_period, 0) AS total_nlp_usage_mt_characters_current_period + FROM user_billing_periods ubp + LEFT JOIN nlp_period_agg na ON ubp.user_id = na.user_id + ) + SELECT + CONCAT(au.id::text, '-', COALESCE(org.id::text, 'orgnone')) AS id, + au.id AS user_id, + org.id AS organization_id, + ued.uid AS extra_details_uid, + au.username, + au.first_name, + au.last_name, + au.email, + au.is_superuser, + au.is_staff, + au.is_active, + TO_CHAR(au.date_joined AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') AS date_joined, + CASE + WHEN au.last_login IS NOT NULL THEN TO_CHAR(au.last_login AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') + ELSE NULL + END AS last_login, + EXISTS ( + SELECT 1 + FROM account_emailaddress aea + WHERE aea.user_id = au.id + AND aea.primary = true + AND aea.verified = true + ) AS validated_email, + ued.validated_password, + EXISTS ( + SELECT 1 + FROM trench_mfamethod mfa + WHERE mfa.user_id = au.id + AND mfa.is_active = true + ) AS mfa_is_active, + EXISTS ( + SELECT 1 + FROM socialaccount_socialaccount sa + WHERE sa.user_id = au.id + ) AS sso_is_active, + EXISTS ( + SELECT 1 + FROM hub_extrauserdetail ued_tos + WHERE ued_tos.user_id = au.id + AND ued_tos.private_data ? 'last_tos_accept_time' + ) AS accepted_tos, + COALESCE( + jsonb_agg( + jsonb_build_object( + 'id', sa.id, + 'provider', sa.provider, + 'uid', sa.uid + ) + ) FILTER (WHERE sa.id IS NOT NULL), + '[]'::jsonb + ) AS social_accounts, + CASE + WHEN org.id IS NOT NULL THEN jsonb_build_object( + 'name', org.name, + 'uid', org.id::text, + 'role', ur.user_role + ) + ELSE NULL + END AS organization, + ued.data::jsonb AS metadata, + COALESCE(ua.total_assets, 0) AS asset_count, + COALESCE(ua.deployed_assets, 0) AS deployed_asset_count, + ucpu.current_period_start, + ucpu.current_period_end, + jsonb_build_object( + 'total_nlp_usage', jsonb_build_object( + 'asr_seconds_current_period', COALESCE(ucpu.total_nlp_usage_asr_seconds_current_period, 0), + 'mt_characters_current_period', COALESCE(ucpu.total_nlp_usage_mt_characters_current_period, 0), + 'asr_seconds_all_time', COALESCE(unl.total_asr_seconds, 0), + 'mt_characters_all_time', COALESCE(unl.total_mt_characters, 0) + ), + 'total_storage_bytes', COALESCE(ubau.total_storage_bytes, 0), + 'total_submission_count', jsonb_build_object( + 'current_period', COALESCE(ubau.total_submission_count_current_period, 0), + 'all_time', COALESCE(ubau.total_submission_count_all_time, 0) + ), + 'balances', jsonb_build_object( + 'submission', + CASE + WHEN ubau.submission_limit IS NULL OR ubau.submission_limit = 0 THEN NULL + ELSE jsonb_build_object( + 'effective_limit', ubau.submission_limit, + 'balance_value', (ubau.submission_limit - COALESCE(ubau.total_submission_count_current_period, 0)), + 'balance_percent', + ((COALESCE(ubau.total_submission_count_current_period, 0)::numeric * 100) / NULLIF(ubau.submission_limit, 0))::int, + 'exceeded', (ubau.submission_limit - COALESCE(ubau.total_submission_count_current_period, 0)) < 0 + ) + END, + 'storage_bytes', + CASE + WHEN ubau.storage_bytes_limit IS NULL OR ubau.storage_bytes_limit = 0 THEN NULL + ELSE jsonb_build_object( + 'effective_limit', ubau.storage_bytes_limit, + 'balance_value', (ubau.storage_bytes_limit - COALESCE(ubau.total_storage_bytes, 0)), + 'balance_percent', + ((COALESCE(ubau.total_storage_bytes, 0)::numeric * 100) / NULLIF(ubau.storage_bytes_limit, 0))::int, + 'exceeded', (ubau.storage_bytes_limit - COALESCE(ubau.total_storage_bytes, 0)) < 0 + ) + END, + 'asr_seconds', + CASE + WHEN ubau.asr_seconds_limit IS NULL OR ubau.asr_seconds_limit = 0 THEN NULL + ELSE jsonb_build_object( + 'effective_limit', ubau.asr_seconds_limit, + 'balance_value', (ubau.asr_seconds_limit - COALESCE(ucpu.total_nlp_usage_asr_seconds_current_period, 0)), + 'balance_percent', + ((COALESCE(ucpu.total_nlp_usage_asr_seconds_current_period, 0)::numeric * 100) / NULLIF(ubau.asr_seconds_limit, 0))::int, + 'exceeded', (ubau.asr_seconds_limit - COALESCE(ucpu.total_nlp_usage_asr_seconds_current_period, 0)) < 0 + ) + END, + 'mt_characters', + CASE + WHEN ubau.mt_characters_limit IS NULL OR ubau.mt_characters_limit = 0 THEN NULL + ELSE jsonb_build_object( + 'effective_limit', ubau.mt_characters_limit, + 'balance_value', (ubau.mt_characters_limit - COALESCE(ucpu.total_nlp_usage_mt_characters_current_period, 0)), + 'balance_percent', + ((COALESCE(ucpu.total_nlp_usage_mt_characters_current_period, 0)::numeric * 100) / NULLIF(ubau.mt_characters_limit, 0))::int, + 'exceeded', (ubau.mt_characters_limit - COALESCE(ucpu.total_nlp_usage_mt_characters_current_period, 0)) < 0 + ) + END + ) + )::jsonb AS service_usage, + {{MV_SUBSCRIPTIONS_SELECT}} + FROM auth_user au + LEFT JOIN organizations_organizationuser ou ON au.id = ou.user_id + LEFT JOIN organizations_organization org ON ou.organization_id = org.id + {{MV_STRIPE_JOINS}} + LEFT JOIN hub_extrauserdetail ued ON au.id = ued.user_id + LEFT JOIN socialaccount_socialaccount sa ON au.id = sa.user_id + LEFT JOIN user_nlp_usage unl ON au.id = unl.user_id + LEFT JOIN user_assets ua ON au.id = ua.user_id + LEFT JOIN user_role_map ur ON ur.user_id = au.id + LEFT JOIN user_current_period_usage ucpu ON au.id = ucpu.user_id AND ucpu.organization_id = org.id + LEFT JOIN user_billing_periods ubau ON au.id = ubau.user_id AND ubau.organization_id = org.id + WHERE au.id != {settings.ANONYMOUS_USER_ID} + GROUP BY + au.id, + au.username, + au.first_name, + au.last_name, + au.email, + au.is_superuser, + au.is_staff, + au.is_active, + ued.uid, + ued.validated_password, + ued.data, + org.id, + org.name, + au.date_joined, + au.last_login, + unl.total_asr_seconds, + unl.total_mt_characters, + ua.total_assets, + ua.deployed_assets, + ur.user_role, + ucpu.current_period_start, + ucpu.current_period_end, + ucpu.total_nlp_usage_asr_seconds_current_period, + ucpu.total_nlp_usage_mt_characters_current_period, + ubau.submission_limit, + ubau.storage_bytes_limit, + ubau.asr_seconds_limit, + ubau.mt_characters_limit, + ubau.total_storage_bytes, + ubau.total_submission_count_all_time, + ubau.total_submission_count_current_period; + """ + +STRIPE_SUBSCRIPTIONS = """ + COALESCE( + jsonb_agg( + jsonb_build_object( + 'items', ( + SELECT jsonb_agg( + jsonb_build_object( + 'id', si.id, + 'price', jsonb_build_object( + 'id', pr.id, + 'nickname', pr.nickname, + 'currency', pr.currency, + 'type', pr.type, + 'recurring', pr.recurring, + 'unit_amount', pr.unit_amount, + 'human_readable_price', + CASE + WHEN pr.recurring IS NOT NULL THEN + CONCAT( + TO_CHAR(pr.unit_amount::numeric / 100, 'FM$999,999.00'), + ' USD/', + pr.recurring->>'interval' + ) + ELSE + TO_CHAR(pr.unit_amount::numeric / 100, 'FM$999,999.00') + END, + 'metadata', pr.metadata, + 'active', pr.active, + 'product', jsonb_build_object( + 'id', prod.id, + 'name', prod.name, + 'description', prod.description, + 'type', prod.type, + 'metadata', prod.metadata + ), + 'transform_quantity', pr.transform_quantity + ), + 'quantity', si.quantity + ) + ) + FROM djstripe_subscriptionitem si + JOIN djstripe_price pr ON si.price_id = pr.djstripe_id + JOIN djstripe_product prod ON pr.product_id = prod.id + WHERE si.subscription_id = sub.id + ), + 'schedule', sub.schedule_id, + 'djstripe_created', sub.djstripe_created, + 'djstripe_updated', sub.djstripe_updated, + 'id', sub.id, + 'livemode', sub.livemode, + 'created', sub.created, + 'metadata', sub.metadata, + 'description', sub.description, + 'application_fee_percent', sub.application_fee_percent, + 'billing_cycle_anchor', sub.billing_cycle_anchor, + 'billing_thresholds', sub.billing_thresholds, + 'cancel_at', sub.cancel_at, + 'cancel_at_period_end', sub.cancel_at_period_end, + 'canceled_at', sub.canceled_at, + 'collection_method', sub.collection_method, + 'current_period_start', sub.current_period_start, + 'current_period_end', sub.current_period_end, + 'days_until_due', sub.days_until_due, + 'discount', sub.discount, + 'ended_at', sub.ended_at, + 'next_pending_invoice_item_invoice', sub.next_pending_invoice_item_invoice, + 'pause_collection', sub.pause_collection, + 'pending_invoice_item_interval', sub.pending_invoice_item_interval, + 'pending_update', sub.pending_update, + 'proration_behavior', sub.proration_behavior, + 'proration_date', sub.proration_date, + 'quantity', sub.quantity, + 'start_date', sub.start_date, + 'status', sub.status, + 'trial_end', sub.trial_end, + 'trial_start', sub.trial_start, + 'djstripe_owner_account', sub.djstripe_owner_account_id, + 'customer', cust.id, + 'default_payment_method', sub.default_payment_method_id, + 'default_source', sub.default_source_id, + 'latest_invoice', sub.latest_invoice_id, + 'pending_setup_intent', sub.pending_setup_intent_id, + 'plan', sub.plan_id, + 'default_tax_rates', ( + SELECT + COALESCE( + jsonb_agg( + jsonb_build_object( + 'id', tax.id, + 'djstripe_id', tax.djstripe_id, + 'description', tax.description, + 'display_name', tax.display_name, + 'inclusive', tax.inclusive, + 'jurisdiction', tax.jurisdiction, + 'percentage', tax.percentage, + 'tax_type', tax.tax_type, + 'active', tax.active, + 'country', tax.country, + 'state', tax.state + ) + ) FILTER (WHERE tax.id IS NOT NULL), + '[]'::jsonb + ) + FROM + djstripe_djstripesubscriptiondefaulttaxrate AS sub_tax_rate + JOIN + djstripe_taxrate AS tax + ON sub_tax_rate.taxrate_id = tax.djstripe_id + WHERE + sub_tax_rate.subscription_id::text = sub.id::text + ) + ) + ) FILTER (WHERE sub.id IS NOT NULL), + '[]'::jsonb + ) AS subscriptions + """ + +NO_STRIPE_SUBSCRIPTIONS = """ + '[]'::jsonb AS subscriptions + """ + +STRIPE_JOINS = """ + LEFT JOIN djstripe_subscription sub ON sub.metadata->>'organization_id' = org.id::text + LEFT JOIN djstripe_customer cust ON sub.customer_id = cust.id + """ + +NO_STRIPE_JOINS = "" + +if settings.STRIPE_ENABLED: + MV_PARAMS = { + 'MV_STRIPE_JOINS': STRIPE_JOINS, + 'MV_SUBSCRIPTIONS_SELECT': STRIPE_SUBSCRIPTIONS, + } +else: + MV_PARAMS = { + 'MV_STRIPE_JOINS': NO_STRIPE_JOINS, + 'MV_SUBSCRIPTIONS_SELECT': NO_STRIPE_SUBSCRIPTIONS, + } + +CREATE_MV_SQL = CREATE_MV_BASE_SQL.format(**MV_PARAMS) + +DROP_MV_SQL = """ + DROP MATERIALIZED VIEW IF EXISTS user_reports_userreportsmv; + """ + +CREATE_INDEXES_SQL = """ + CREATE UNIQUE INDEX IF NOT EXISTS idx_user_reports_mv_id ON user_reports_userreportsmv (id); + CREATE UNIQUE INDEX IF NOT EXISTS idx_user_reports_mv_user_org ON user_reports_userreportsmv (user_id, organization_id); + """ +DROP_INDEXES_SQL = """ + DROP INDEX IF EXISTS idx_user_reports_mv_user_org; + DROP INDEX IF EXISTS idx_user_reports_mv_id; + """ + + +def manually_create_mv_instructions(apps, schema_editor): + print( + f""" + ⚠️ ATTENTION ⚠️ + Run the SQL query below in PostgreSQL directly to create the materialized view: + + {CREATE_MV_SQL} + + Then run the SQL query below to create the indexes: + + {CREATE_INDEXES_SQL} + + """.replace('CREATE UNIQUE INDEX', 'CREATE UNIQUE INDEX CONCURRENTLY') + ) + + +def manually_drop_mv_instructions(apps, schema_editor): + print( + f""" + ⚠️ ATTENTION ⚠️ + Run the SQL query below in PostgreSQL directly: + + {DROP_MV_SQL} + + """ + ) + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [ + ('user_reports', '0001_initial'), + ('trackers', '0005_remove_year_and_month'), + ('mfa', '0004_alter_mfamethod_date_created_and_more'), + ] + + if settings.SKIP_HEAVY_MIGRATIONS: + operations = [ + migrations.RunPython( + manually_create_mv_instructions, + manually_drop_mv_instructions, + ) + ] + else: + operations = [ + migrations.RunSQL( + sql=CREATE_MV_SQL, + reverse_sql=DROP_MV_SQL, + ), + migrations.RunSQL( + sql=CREATE_INDEXES_SQL, + reverse_sql=DROP_INDEXES_SQL, + ), + ] diff --git a/kobo/apps/user_reports/migrations/__init__.py b/kobo/apps/user_reports/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/kobo/apps/user_reports/models.py b/kobo/apps/user_reports/models.py new file mode 100644 index 0000000000..8d92bf0ac0 --- /dev/null +++ b/kobo/apps/user_reports/models.py @@ -0,0 +1,128 @@ +from django.db import models +from django.db.models import Q + +from kpi.fields import KpiUidField +from kpi.models.abstract_models import AbstractTimeStampedModel + + +class BillingAndUsageSnapshotStatus(models.TextChoices): + IN_PROGRESS = 'in_progress' + COMPLETED = 'completed' + ABORTED = 'aborted' + + +class BillingAndUsageSnapshot(AbstractTimeStampedModel): + """ + A snapshot table for storing precomputed organization billing dates, + submission counts, and storage usage data. + + Why this table exists: + 1. Maintaining billing period calculations directly inside the materialized view + would make it too complex and hard to manage. + 2. Usage data such as total submissions, current period submissions, and storage + resides in the `kobocat` db, while the materialized view lives in the `kpi` + db. Joining across databases for 1.7M+ users would be inefficient. + 3. A periodic Celery task (`refresh_user_report_snapshots`) precomputes these + values and writes them here. The materialized view then joins against this + table efficiently. + """ + + organization = models.OneToOneField( + 'organizations.Organization', on_delete=models.CASCADE + ) + effective_user_id = models.IntegerField(null=True, blank=True, db_index=True) + total_storage_bytes = models.BigIntegerField(default=0) + total_submission_count_all_time = models.BigIntegerField(default=0) + total_submission_count_current_period = models.BigIntegerField(default=0) + billing_period_start = models.DateTimeField(null=True, blank=True) + billing_period_end = models.DateTimeField(null=True, blank=True) + last_snapshot_run = models.ForeignKey( + 'user_reports.BillingAndUsageSnapshotRun', + related_name='snapshots', + on_delete=models.CASCADE, + ) + submission_limit = models.BigIntegerField(null=True, blank=True) + storage_bytes_limit = models.BigIntegerField(null=True, blank=True) + asr_seconds_limit = models.BigIntegerField(null=True, blank=True) + mt_characters_limit = models.BigIntegerField(null=True, blank=True) + + class Meta: + indexes = [ + models.Index(fields=['effective_user_id'], name='idx_bau_user'), + models.Index(fields=['date_created'], name='idx_bau_created'), + ] + + constraints = [ + models.UniqueConstraint( + fields=['organization'], name='uniq_snapshot_per_org' + ), + ] + + def __str__(self): + return f'BillingAndUsageSnapshot(org={self.organization_id})' + + +class BillingAndUsageSnapshotRun(AbstractTimeStampedModel): + """ + A snapshot run table to track the progress and status of the + `refresh_user_report_snapshots` Celery task. + """ + + uid = KpiUidField('busr') + status = models.CharField( + max_length=32, + choices=BillingAndUsageSnapshotStatus.choices, + default=BillingAndUsageSnapshotStatus.IN_PROGRESS, + ) + last_processed_org_id = models.CharField(null=True, blank=True) + details = models.JSONField(null=True, blank=True) + singleton = models.BooleanField(default=True, editable=False) + + class Meta: + ordering = ['-date_created'] + indexes = [ + models.Index( + fields=['status', 'date_modified'], name='idx_bau_run_status_expires' + ), + ] + constraints = [ + models.UniqueConstraint( + fields=('singleton',), + condition=Q(status=BillingAndUsageSnapshotStatus.IN_PROGRESS), + name='uniq_run_in_progress', + ), + ] + + +class UserReports(models.Model): + id = models.CharField(primary_key=True, max_length=80) + extra_details_uid = models.CharField(null=True, blank=True) + username = models.CharField() + first_name = models.CharField() + last_name = models.CharField() + email = models.EmailField() + is_superuser = models.BooleanField() + is_staff = models.BooleanField() + is_active = models.BooleanField() + date_joined = models.CharField() + last_login = models.CharField(null=True, blank=True) + validated_email = models.BooleanField() + validated_password = models.BooleanField() + mfa_is_active = models.BooleanField() + sso_is_active = models.BooleanField() + accepted_tos = models.BooleanField() + social_accounts = models.JSONField(default=list) + organization = models.JSONField(null=True, blank=True) + metadata = models.JSONField(null=True, blank=True) + subscriptions = models.JSONField(default=list) + + asset_count = models.IntegerField(default=0) + deployed_asset_count = models.IntegerField(default=0) + + current_period_start = models.DateTimeField(null=True, blank=True) + current_period_end = models.DateTimeField(null=True, blank=True) + service_usage = models.JSONField(null=True, blank=True) + + class Meta: + managed = False + db_table = 'user_reports_userreportsmv' diff --git a/kobo/apps/user_reports/seralizers.py b/kobo/apps/user_reports/seralizers.py new file mode 100644 index 0000000000..d9f3c4f784 --- /dev/null +++ b/kobo/apps/user_reports/seralizers.py @@ -0,0 +1,104 @@ +from typing import Any + +from django.utils import timezone +from rest_framework import serializers + +from kobo.apps.organizations.constants import UsageType +from kobo.apps.organizations.models import Organization +from kobo.apps.stripe.utils.subscription_limits import ( + get_organizations_effective_limits, +) +from kobo.apps.user_reports.models import UserReports +from kpi.utils.usage_calculator import ( + calculate_usage_balance, +) + + +class UserReportsSerializer(serializers.ModelSerializer): + extra_details__uid = serializers.CharField( + source='extra_details_uid', read_only=True + ) + service_usage = serializers.SerializerMethodField() + account_restricted = serializers.SerializerMethodField() + + class Meta: + model = UserReports + fields = [ + 'extra_details__uid', + 'username', + 'first_name', + 'last_name', + 'email', + 'is_superuser', + 'is_staff', + 'is_active', + 'date_joined', + 'last_login', + 'validated_email', + 'validated_password', + 'mfa_is_active', + 'sso_is_active', + 'accepted_tos', + 'social_accounts', + 'organization', + 'metadata', + 'subscriptions', + 'service_usage', + 'account_restricted', + 'asset_count', + 'deployed_asset_count', + ] + + def get_account_restricted(self, obj) -> bool: + service_usage = obj.service_usage + balances = service_usage.get('balances', {}) + return any(balance and balance.get('exceeded') for balance in balances.values()) + + def get_service_usage(self, obj) -> dict[str, Any]: + su = obj.service_usage + + # Format billing period dates + current_period_start = None + current_period_end = None + if obj.current_period_start: + current_period_start = obj.current_period_start.isoformat() + if obj.current_period_end: + current_period_end = obj.current_period_end.isoformat() + + su['current_period_start'] = current_period_start + su['current_period_end'] = current_period_end + su['last_updated'] = timezone.now().isoformat() + return su + + def _calculate_usage_balances(self, obj) -> dict[str, Any]: + """ + Calculate usage balances against organization limits. + + This is the only remaining runtime calculation, but it's much more + efficient since all usage data is pre-computed. + """ + if not obj.organization_id: + return {} + + organization = Organization.objects.get(id=obj.organization_id) + limits = get_organizations_effective_limits([organization], True, True) + org_limits = limits.get(organization.id, {}) + + return { + 'submission': calculate_usage_balance( + limit=org_limits.get(f'{UsageType.SUBMISSION}_limit', float('inf')), + usage=obj.total_submission_count_current_period, + ), + 'storage_bytes': calculate_usage_balance( + limit=org_limits.get(f'{UsageType.STORAGE_BYTES}_limit', float('inf')), + usage=obj.total_storage_bytes, + ), + 'asr_seconds': calculate_usage_balance( + limit=org_limits.get(f'{UsageType.ASR_SECONDS}_limit', float('inf')), + usage=obj.total_nlp_usage_asr_seconds_current_period, + ), + 'mt_characters': calculate_usage_balance( + limit=org_limits.get(f'{UsageType.MT_CHARACTERS}_limit', float('inf')), + usage=obj.total_nlp_usage_mt_characters_current_period, + ), + } diff --git a/kobo/apps/user_reports/tasks.py b/kobo/apps/user_reports/tasks.py new file mode 100644 index 0000000000..df9763d0fe --- /dev/null +++ b/kobo/apps/user_reports/tasks.py @@ -0,0 +1,108 @@ +from django.conf import settings +from django.core.cache import cache +from django.utils import timezone + +from kobo.apps.stripe.utils.billing_dates import get_current_billing_period_dates_by_org +from kobo.apps.stripe.utils.subscription_limits import ( + get_organizations_effective_limits +) +from kobo.apps.user_reports.models import ( + BillingAndUsageSnapshotRun, + BillingAndUsageSnapshotStatus, +) +from kobo.apps.user_reports.utils.billing_and_usage_calculator import ( + BillingAndUsageCalculator +) +from kobo.apps.user_reports.utils.snapshot_refresh_helpers import ( + cleanup_stale_snapshots_and_refresh_mv, + get_or_create_run, + iter_org_chunks_after, + process_chunk, +) +from kobo.celery import celery_app +from kpi.utils.log import logging + + +@celery_app.task( + queue='kpi_low_priority_queue', + soft_time_limit=settings.CELERY_LONG_RUNNING_TASK_SOFT_TIME_LIMIT, + time_limit=settings.CELERY_LONG_RUNNING_TASK_TIME_LIMIT, +) +def refresh_user_report_snapshots(**kwargs): + """ + Refresh `BillingAndUsageSnapshot` table in batches + + Core Features: + - Redis Lock: + Prevents concurrent workers from running this task at the same time. + If a lock already exists, the task exits immediately. + - Snapshot Run Tracking (`BillingAndUsageSnapshotRun`): + Tracks the progress of each run (status, last_processed_org_id, details). + Allows the task to resume from where it left off after failure or + pod restarts. + - Incremental Batching: + Uses key-set pagination to process organizations in ordered chunks + without performance penalties (no OFFSET). + + Workflow: + 1. Acquire a non-blocking Redis lock (`billing_and_usage_snapshot:run_lock`) + with TTL = hard time limit + safety margin. + - If lock not acquired; exit (another worker is already processing). + 2. Fetch or create an active snapshot run (status = 'running'): + - If no active run exists, create a new one (cursor reset). + - If exists, resume from `last_processed_org_id`. + 3. Iterate organizations in key-set chunks: + - Compute usage data for the batch using `BillingAndUsageCalculator`. + - Upsert (`bulk_update` + `bulk_create`) `BillingAndUsageSnapshot` records + for each organization. + - Persist progress: update `last_processed_org_id` in the run. + 4. If the task is killed or hits the time limit: + - Partial progress (up to the last committed chunk) is safely stored. + - On the next run, task resumes from where it stopped. + 5. After all organizations processed: + - Delete stale snapshot rows (not updated in this run). + - Refresh the `user_reports_userreportsmv` materialized view concurrently. + - Mark the run as 'completed'. + """ + calc = BillingAndUsageCalculator() + cache_key = 'billing_and_usage_snapshot:run_lock' + lock_timeout = settings.CELERY_LONG_RUNNING_TASK_TIME_LIMIT + 60 + lock = cache.lock(cache_key, timeout=lock_timeout) + if not lock.acquire(blocking=False, blocking_timeout=0): + logging.info('Nothing to do, task is already running!') + return + + # Claim the existing snapshot run or create a new one + run = get_or_create_run() + last_processed_org_id = run.last_processed_org_id or '' + try: + while chunk_qs := iter_org_chunks_after(last_processed_org_id): + billing_map = get_current_billing_period_dates_by_org(chunk_qs) + limits_map = get_organizations_effective_limits(chunk_qs, True, True) + usage_map = calc.calculate_usage_batch(chunk_qs, billing_map) + last_processed_org_id = process_chunk( + chunk_qs, usage_map, limits_map, run.pk + ) + + # Update the run progress + BillingAndUsageSnapshotRun.objects.filter(pk=run.pk).update( + last_processed_org_id=last_processed_org_id, + date_modified=timezone.now(), + ) + + # All orgs processed: cleanup stale, refresh MV and mark run as completed + cleanup_stale_snapshots_and_refresh_mv(run.pk) + BillingAndUsageSnapshotRun.objects.filter(pk=run.pk).update( + status=BillingAndUsageSnapshotStatus.COMPLETED, + date_modified=timezone.now(), + ) + + # Release the lock + lock.release() + + except Exception as ex: + run = BillingAndUsageSnapshotRun.objects.get(pk=run.pk) + details = run.details or {} + details.update({'last_error': str(ex), 'ts': timezone.now().isoformat()}) + run.details = details + run.save(update_fields=['details', 'date_modified']) diff --git a/kobo/apps/user_reports/tests/test_user_reports.py b/kobo/apps/user_reports/tests/test_user_reports.py new file mode 100644 index 0000000000..cdf2476523 --- /dev/null +++ b/kobo/apps/user_reports/tests/test_user_reports.py @@ -0,0 +1,305 @@ +from unittest.mock import patch + +import pytest +from django.conf import settings +from django.core.cache import cache +from django.db import connection +from django.urls import reverse +from django.utils import timezone +from model_bakery import baker +from rest_framework import status + +from kobo.apps.kobo_auth.shortcuts import User +from kobo.apps.openrosa.apps.logger.models import DailyXFormSubmissionCounter +from kobo.apps.openrosa.apps.main.models import UserProfile +from kobo.apps.organizations.constants import UsageType +from kobo.apps.trackers.models import NLPUsageCounter +from kobo.apps.user_reports.models import BillingAndUsageSnapshot +from kobo.apps.user_reports.tasks import refresh_user_report_snapshots +from kobo.apps.user_reports.utils.snapshot_refresh_helpers import ( + refresh_user_reports_materialized_view, +) +from kpi.tests.base_test_case import BaseTestCase + + +class UserReportsViewSetAPITestCase(BaseTestCase): + fixtures = ['test_data'] + + def setUp(self): + self.client.login(username='adminuser', password='pass') + self.url = reverse(self._get_endpoint('api_v2:user-reports-list')) + + self.someuser = User.objects.get(username='someuser') + self.organization = self.someuser.organization + + baker.make(BillingAndUsageSnapshot, organization_id=self.organization.id) + + refresh_user_reports_materialized_view(concurrently=False) + + def test_list_view_requires_authentication(self): + self.client.logout() + response = self.client.get(self.url) + self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) + + def test_list_view_requires_superuser_permission(self): + self.client.logout() + self.client.force_login(user=self.someuser) + response = self.client.get(self.url) + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + def test_list_view_succeeds_for_superuser(self): + response = self.client.get(self.url) + self.assertEqual(response.status_code, status.HTTP_200_OK) + # Make sure that all 3 users from the 'test_data' are included + self.assertEqual(len(response.data['results']), 3) + + def test_endpoint_returns_error_when_mv_is_missing(self): + # Drop the materialized view before the test + with connection.cursor() as cursor: + cursor.execute( + 'DROP MATERIALIZED VIEW IF EXISTS user_reports_userreportsmv CASCADE;' + ) + + response = self.client.get(self.url) + + self.assertEqual(response.status_code, status.HTTP_503_SERVICE_UNAVAILABLE) + self.assertEqual( + response.json(), + { + 'details': 'The data source for user reports is missing. ' + 'Please run 0002_create_user_reports_mv to create the ' + 'materialized view: user_reports_userreportsmv.', + }, + ) + + @pytest.mark.skipif( + not settings.STRIPE_ENABLED, reason='Requires stripe functionality' + ) + def test_subscription_data_is_correctly_returned(self): + + # Create and add a subscription to someuser + from djstripe.enums import BillingScheme + from djstripe.models import Customer + + self.customer = baker.make(Customer, subscriber=self.organization) + self.subscription = baker.make( + 'djstripe.Subscription', + customer=self.customer, + items__price__livemode=False, + items__price__billing_scheme=BillingScheme.per_unit, + livemode=False, + metadata={'organization_id': str(self.organization.id)}, + ) + refresh_user_reports_materialized_view(concurrently=False) + user_with_sub = self._get_someuser_data() + self.assertEqual(len(user_with_sub['subscriptions']), 1) + self.assertEqual(user_with_sub['subscriptions'][0]['id'], self.subscription.id) + + subscription_item = user_with_sub['subscriptions'][0]['items'][0] + + self.assertEqual(subscription_item['id'], self.subscription.items.first().id) + self.assertEqual( + subscription_item['price']['id'], self.subscription.items.first().price.id + ) + self.assertEqual( + subscription_item['price']['product']['id'], + self.subscription.items.first().price.product.id, + ) + self.assertEqual( + user_with_sub['subscriptions'][0]['customer'], self.customer.id + ) + self.assertEqual( + user_with_sub['subscriptions'][0]['metadata']['organization_id'], + self.subscription.metadata['organization_id'], + ) + + def test_service_usage_data_is_correctly_returned(self): + """ + Test that the service usage data is correctly calculated and returned + in the user report, including balances based on mocked limits + """ + # Create submission counter entries to simulate usage + DailyXFormSubmissionCounter.objects.create( + user_id=self.someuser.id, + date=timezone.now().date(), + counter=15 + ) + DailyXFormSubmissionCounter.objects.create( + user_id=self.someuser.id, + date=timezone.now().date() - timezone.timedelta(days=100), + counter=135 + ) + NLPUsageCounter.objects.create( + user_id=self.someuser.id, + date=timezone.now().date(), + total_asr_seconds=100 + ) + NLPUsageCounter.objects.create( + user_id=self.someuser.id, + date=timezone.now().date() - timezone.timedelta(days=100), + total_asr_seconds=80, + ) + UserProfile.objects.filter(user_id=self.someuser.id).update( + attachment_storage_bytes=200000000 + ) + + # Mock `get_organizations_effective_limits` to return test limits. + mock_limits = { + self.someuser.organization.id: { + f'{UsageType.SUBMISSION}_limit': 10, + f'{UsageType.STORAGE_BYTES}_limit': 500000000, + f'{UsageType.ASR_SECONDS}_limit': 120, + f'{UsageType.MT_CHARACTERS}_limit': 5, + } + } + with patch( + 'kobo.apps.user_reports.tasks.get_organizations_effective_limits', + return_value=mock_limits + ): + cache.clear() + refresh_user_report_snapshots() + self.client.login(username='adminuser', password='pass') + someuser_data = self._get_someuser_data() + + service_usage = someuser_data['service_usage'] + # Assert total usage counts from the snapshot + self.assertEqual(service_usage['total_submission_count']['current_period'], 15) + self.assertEqual(service_usage['total_submission_count']['all_time'], 150) + self.assertEqual(service_usage['total_storage_bytes'], 200000000) + self.assertEqual( + service_usage['total_nlp_usage']['asr_seconds_current_period'], 100 + ) + self.assertEqual(service_usage['total_nlp_usage']['asr_seconds_all_time'], 180) + self.assertEqual( + service_usage['total_nlp_usage']['mt_characters_current_period'], 0 + ) + self.assertEqual(service_usage['total_nlp_usage']['mt_characters_all_time'], 0) + + # Assert calculated balances based on mock limits and real results + balances = service_usage['balances'] + + # Submission balance: 15 / 10 = 1.5, so 150% and exceeded. + self.assertIsNotNone(balances['submission']) + self.assertTrue(balances['submission']['exceeded']) + self.assertEqual(balances['submission']['effective_limit'], 10) + self.assertEqual(balances['submission']['balance_value'], -5) + self.assertEqual(balances['submission']['balance_percent'], 150) + + # Storage balance: 200,000,000 / 500,000,000 = 0.4, so 40% and not exceeded. + self.assertIsNotNone(balances['storage_bytes']) + self.assertFalse(balances['storage_bytes']['exceeded']) + self.assertEqual(balances['storage_bytes']['effective_limit'], 500000000) + self.assertEqual(balances['storage_bytes']['balance_value'], 300000000) + self.assertEqual(balances['storage_bytes']['balance_percent'], 40) + + # ASR Seconds balance: 0 / 120 = 0, so 0% and not exceeded. + self.assertIsNotNone(balances['asr_seconds']) + self.assertFalse(balances['asr_seconds']['exceeded']) + self.assertEqual(balances['asr_seconds']['effective_limit'], 120) + self.assertEqual(balances['asr_seconds']['balance_value'], 20) + self.assertEqual(balances['asr_seconds']['balance_percent'], 83) + + # MT Characters balance: 0 / 5 = 0, so 0% and not exceeded. + self.assertIsNotNone(balances['mt_characters']) + self.assertFalse(balances['mt_characters']['exceeded']) + self.assertEqual(balances['mt_characters']['effective_limit'], 5) + self.assertEqual(balances['mt_characters']['balance_value'], 5) + self.assertEqual(balances['mt_characters']['balance_percent'], 0) + + def test_organization_data_is_correctly_returned(self): + someuser_data = self._get_someuser_data() + + organization_data = someuser_data['organization'] + + self.assertEqual( + organization_data['name'], self.someuser.organization.name + ) + self.assertEqual( + organization_data['uid'], str(self.someuser.organization.id) + ) + self.assertEqual(organization_data['role'], 'owner') + + def test_account_restricted_field(self): + # Verify `account_restricted` is initially false + response = self.client.get(self.url) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + results = response.data['results'] + someuser_data = next( + (user for user in results if user['username'] == 'someuser'), + None, + ) + + self.assertIsNotNone(someuser_data) + self.assertFalse(someuser_data['account_restricted']) + + # Create a submission counter entry to simulate usage + DailyXFormSubmissionCounter.objects.create( + user_id=self.someuser.id, + date=timezone.now().date(), + counter=10 + ) + + # Mock the `get_organizations_effective_limits` function + # to return a predefined limit + mock_limits = { + self.someuser.organization.id: { + f'{UsageType.SUBMISSION}_limit': 1, + f'{UsageType.STORAGE_BYTES}_limit': 500000000, + f'{UsageType.ASR_SECONDS}_limit': 120, + f'{UsageType.MT_CHARACTERS}_limit': 5, + } + } + with patch( + 'kobo.apps.user_reports.tasks.get_organizations_effective_limits', + return_value=mock_limits + ): + cache.clear() + refresh_user_report_snapshots() + + self.client.login(username='adminuser', password='pass') + someuser_data = self._get_someuser_data() + self.assertTrue(someuser_data['account_restricted']) + + def test_accepted_tos_field(self): + # Verify `accepted_tos` is initially false + response = self.client.get(self.url) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + results = response.data['results'] + self.assertEqual(results[0]['accepted_tos'], False) + + # POST to the tos endpoint to accept the terms of service + tos_url = reverse(self._get_endpoint('tos')) + response = self.client.post(tos_url) + assert response.status_code == status.HTTP_204_NO_CONTENT + + refresh_user_reports_materialized_view(concurrently=False) + + # Verify `accepted_tos` has been set to True + response = self.client.get(self.url) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + results = response.data['results'] + self.assertTrue(results[0]['accepted_tos']) + + def test_ordering_by_date_joined(self): + response = self.client.get(self.url, {'ordering': 'date_joined'}) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + results = response.data['results'] + self.assertEqual(results[0]['username'], 'adminuser') + self.assertEqual(results[1]['username'], 'someuser') + self.assertEqual(results[2]['username'], 'anotheruser') + + def _get_someuser_data(self): + + response = self.client.get(self.url) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + results = response.data['results'] + someuser_data = next( + (user for user in results if user['username'] == 'someuser'), None + ) + self.assertIsNotNone(someuser_data) + return someuser_data diff --git a/kobo/apps/user_reports/typing_aliases.py b/kobo/apps/user_reports/typing_aliases.py new file mode 100644 index 0000000000..26981ba4ab --- /dev/null +++ b/kobo/apps/user_reports/typing_aliases.py @@ -0,0 +1,8 @@ +from typing import Optional + +from django.db.models import QuerySet + +from kobo.apps.organizations.models import Organization + + +OrganizationIterator = Optional[QuerySet[Organization] | list[Organization]] diff --git a/kobo/apps/user_reports/utils/billing_and_usage_calculator.py b/kobo/apps/user_reports/utils/billing_and_usage_calculator.py new file mode 100644 index 0000000000..e564706ff6 --- /dev/null +++ b/kobo/apps/user_reports/utils/billing_and_usage_calculator.py @@ -0,0 +1,89 @@ +from django.db.models import Q, Sum +from django.db.models.functions import Coalesce + +from kobo.apps.openrosa.apps.logger.models import DailyXFormSubmissionCounter +from kobo.apps.organizations.models import Organization +from kpi.utils.usage_calculator import get_storage_usage_by_user_id +from ..typing_aliases import OrganizationIterator + + +class BillingAndUsageCalculator: + + def calculate_usage_batch( + self, organizations: OrganizationIterator, billing_dates: dict + ) -> dict: + org_map = {} + for org in organizations: + if not (eff_uid := self.get_effective_user_id(org)): + pass + + org_map[org.id] = { + 'effective_user_id': eff_uid, + 'billing_dates': billing_dates.get(org.id, {}), + } + + user_ids = [v['effective_user_id'] for v in org_map.values()] + storage_map = get_storage_usage_by_user_id(user_ids) + submission_map = self._get_submission_usage_batch( + user_ids, + {v['effective_user_id']: v['billing_dates'] for v in org_map.values()}, + ) + + result = {} + for org_id, info in org_map.items(): + uid = info['effective_user_id'] + result[org_id] = { + 'effective_user_id': uid, + 'total_storage_bytes': storage_map.get(uid, 0), + 'total_submission_count_all_time': submission_map.get(uid, {}).get( + 'all_time', 0 + ), + 'total_submission_count_current_period': submission_map.get( + uid, {} + ).get( + 'current_period', 0 + ), + 'billing_period_start': info['billing_dates'].get('start'), + 'billing_period_end': info['billing_dates'].get('end'), + } + return result + + def get_effective_user_id(self, organization: Organization) -> int | None: + try: + return organization.owner_user_object.pk + except AttributeError: + return None + + def _get_submission_usage_batch(self, user_ids, date_ranges_by_user): + if not user_ids: + return {} + + # Get all-time submission counts + rows = ( + DailyXFormSubmissionCounter.objects.filter(user_id__in=user_ids) + .values('user_id') + .annotate(total=Coalesce(Sum('counter'), 0)) + ) + all_time = {r['user_id']: r['total'] for r in rows} + + combined_q = Q() + for uid, dr in date_ranges_by_user.items(): + if dr.get('start') and dr.get('end'): + combined_q |= Q(user_id=uid, date__range=[dr['start'], dr['end']]) + + current = {} + if combined_q: + rows = ( + DailyXFormSubmissionCounter.objects.filter(combined_q) + .values('user_id') + .annotate(total=Coalesce(Sum('counter'), 0)) + ) + current = {r['user_id']: r['total'] for r in rows} + + return { + uid: { + 'all_time': all_time.get(uid, 0), + 'current_period': current.get(uid, 0), + } + for uid in user_ids + } diff --git a/kobo/apps/user_reports/utils/snapshot_refresh_helpers.py b/kobo/apps/user_reports/utils/snapshot_refresh_helpers.py new file mode 100644 index 0000000000..891b7faabe --- /dev/null +++ b/kobo/apps/user_reports/utils/snapshot_refresh_helpers.py @@ -0,0 +1,134 @@ +from math import inf + +from django.db import connection +from django.db.models import Q +from django.db.models.query import QuerySet + +from kobo.apps.organizations.models import Organization +from kobo.apps.user_reports.models import ( + BillingAndUsageSnapshot, + BillingAndUsageSnapshotRun, + BillingAndUsageSnapshotStatus, +) +from ..typing_aliases import OrganizationIterator + +CHUNK_SIZE = 1000 + + +def cleanup_stale_snapshots_and_refresh_mv(run_id: str): + """ + Delete stale snapshot rows and refresh the materialized view + """ + while True: + stale_ids = list( + BillingAndUsageSnapshot.objects.filter( + ~Q(last_snapshot_run_id=run_id) + ).values_list('pk', flat=True)[:CHUNK_SIZE] + ) + if not stale_ids: + break + BillingAndUsageSnapshot.objects.filter(pk__in=stale_ids).delete() + + refresh_user_reports_materialized_view() + + +def get_or_create_run(): + """ + Get or create a `BillingAndUsageSnapshotRun` with status `IN_PROGRESS` + """ + run, _ = BillingAndUsageSnapshotRun.objects.get_or_create( + status=BillingAndUsageSnapshotStatus.IN_PROGRESS, + ) + return run + + +def iter_org_chunks_after(last_processed_org_id: str) -> QuerySet[Organization]: + """ + Iterate organizations in key set chunks + """ + return Organization.objects.filter(pk__gt=last_processed_org_id).order_by( + 'pk' + )[:CHUNK_SIZE] + + +def process_chunk( + chunk_qs: OrganizationIterator, usage_map: dict, limits_map: dict, run_id: int +) -> str | None: + """ + Apply usage data for a chunk of organizations and persist changes + + For each organization in the chunk: + - If a snapshot already exists, update it with the latest usage data. + - If no snapshot exists, create a new entry. + + Returns the last processed organization ID + """ + + objs = [] + last_org_id = None + + for org_id in chunk_qs.values_list('id', flat=True): + last_org_id = org_id + d = usage_map.get(org_id, {}) + org_limits = limits_map.get(org_id, {}) + + objs.append(BillingAndUsageSnapshot( + organization_id=org_id, + effective_user_id=d.get('effective_user_id'), + total_storage_bytes=d.get('total_storage_bytes', 0), + total_submission_count_all_time=d.get('total_submission_count_all_time', 0), + total_submission_count_current_period=d.get( + 'total_submission_count_current_period', 0 + ), + billing_period_start=d.get('billing_period_start'), + billing_period_end=d.get('billing_period_end'), + last_snapshot_run_id=run_id, + submission_limit=_normalize_limit(org_limits.get('submission_limit')), + storage_bytes_limit=_normalize_limit(org_limits.get('storage_bytes_limit')), + asr_seconds_limit=_normalize_limit(org_limits.get('asr_seconds_limit')), + mt_characters_limit=_normalize_limit(org_limits.get('mt_characters_limit')), + )) + + if objs: + BillingAndUsageSnapshot.objects.bulk_create( + objs, + update_conflicts=True, + update_fields=[ + 'effective_user_id', + 'total_storage_bytes', + 'total_submission_count_all_time', + 'total_submission_count_current_period', + 'billing_period_start', + 'billing_period_end', + 'last_snapshot_run_id', + 'submission_limit', + 'storage_bytes_limit', + 'asr_seconds_limit', + 'mt_characters_limit', + ], + unique_fields=['organization_id'], + ) + + return last_org_id + + +def refresh_user_reports_materialized_view(concurrently=True): + """ + Refreshes the user reports materialized view (optionally concurrently) + """ + concurrent_keyword = ' CONCURRENTLY' if concurrently else '' + sql = f'REFRESH MATERIALIZED VIEW{concurrent_keyword} user_reports_userreportsmv;' + + with connection.cursor() as cursor: + cursor.execute(sql) + + +def _normalize_limit(limit: int | float | None) -> int | None: + """ + Normalize limit values for database storage + """ + if limit is None: + return None + if limit == inf: + return None + return int(limit) diff --git a/kobo/apps/user_reports/views.py b/kobo/apps/user_reports/views.py new file mode 100644 index 0000000000..021938a063 --- /dev/null +++ b/kobo/apps/user_reports/views.py @@ -0,0 +1,81 @@ +from django.db import ProgrammingError +from django_filters.rest_framework import DjangoFilterBackend +from drf_spectacular.utils import extend_schema, extend_schema_view +from rest_framework import mixins, status, viewsets +from rest_framework.filters import OrderingFilter +from rest_framework.response import Response + +from kobo.apps.audit_log.permissions import SuperUserPermission +from kobo.apps.user_reports.models import UserReports +from kobo.apps.user_reports.seralizers import UserReportsSerializer +from kpi.filters import SearchFilter +from kpi.paginators import LimitOffsetPagination +from kpi.permissions import IsAuthenticated +from kpi.schema_extensions.v2.user_reports.serializers import UserReportsListResponse +from kpi.utils.schema_extensions.markdown import read_md +from kpi.utils.schema_extensions.response import open_api_200_ok_response + + +@extend_schema( + tags=['Server logs (superusers)'], +) +@extend_schema_view( + list=extend_schema( + description=read_md('kpi', 'user_reports/list.md'), + responses=open_api_200_ok_response( + UserReportsListResponse, + require_auth=False, + raise_not_found=False, + ), + ), +) +class UserReportsViewSet(mixins.ListModelMixin, viewsets.GenericViewSet): + """ + Available actions: + - list → GET /api/v2/user-reports/ + + Documentation: + - docs/api/v2/users_reports/list.md + """ + + queryset = UserReports.objects.all() + serializer_class = UserReportsSerializer + pagination_class = LimitOffsetPagination + permission_classes = (IsAuthenticated, SuperUserPermission) + filter_backends = [DjangoFilterBackend, OrderingFilter, SearchFilter] + + search_default_field_lookups = [ + 'username__icontains' + ] + + ordering_fields = [ + 'username', + 'email', + 'date_joined', + 'last_login', + 'total_storage_bytes', + 'total_submission_count_current_period', + 'total_submission_count_all_time', + 'total_nlp_usage_asr_seconds_all_time', + 'total_nlp_usage_mt_characters_all_time', + 'asset_count', + 'deployed_asset_count', + ] + ordering = ['username'] + search_fields = ['username', 'email', 'first_name', 'last_name'] + skip_distinct = True + + def list(self, request, *args, **kwargs): + try: + return super().list(request, *args, **kwargs) + except ProgrammingError as e: + if 'relation "user_reports_userreportsmv" does not exist' in str(e): # noqa + return Response( + { + 'details': 'The data source for user reports is missing. ' + 'Please run 0002_create_user_reports_mv to create the ' + 'materialized view: user_reports_userreportsmv.', + }, + status=status.HTTP_503_SERVICE_UNAVAILABLE, + ) + raise e diff --git a/kobo/settings/base.py b/kobo/settings/base.py index 4e8781b66d..730f71850d 100644 --- a/kobo/settings/base.py +++ b/kobo/settings/base.py @@ -147,6 +147,7 @@ 'kobo.apps.openrosa.libs', 'kobo.apps.project_ownership.app.ProjectOwnershipAppConfig', 'kobo.apps.long_running_migrations.app.LongRunningMigrationAppConfig', + 'kobo.apps.user_reports.apps.UserReportsConfig', 'drf_spectacular', ) @@ -1109,7 +1110,7 @@ def __init__(self, *args, **kwargs): 'description': 'Subscribe to and manage shared library collections', }, { - 'name': 'Audit logs (superusers)', + 'name': 'Server logs (superusers)', 'description': 'View server-wide logs', }, { @@ -1443,6 +1444,12 @@ def dj_stripe_request_callback_method(): 'schedule': crontab(minute='*/30'), 'options': {'queue': 'kpi_low_priority_queue'} }, + # Schedule every 30 minutes + 'refresh-user-report-snapshot': { + 'task': 'kobo.apps.user_reports.tasks.refresh_user_report_snapshots', + 'schedule': crontab(minute='*/30'), + 'options': {'queue': 'kpi_low_priority_queue'} + }, # Schedule every day at midnight UTC 'project-ownership-garbage-collector': { 'task': 'kobo.apps.project_ownership.tasks.garbage_collector', diff --git a/kpi/docs/api/v2/user_reports/list.md b/kpi/docs/api/v2/user_reports/list.md new file mode 100644 index 0000000000..755951d294 --- /dev/null +++ b/kpi/docs/api/v2/user_reports/list.md @@ -0,0 +1 @@ +## List user reports diff --git a/kpi/schema_extensions/imports.py b/kpi/schema_extensions/imports.py index 8d6bef6fdd..a1f6a8695b 100644 --- a/kpi/schema_extensions/imports.py +++ b/kpi/schema_extensions/imports.py @@ -20,5 +20,6 @@ import kpi.schema_extensions.v2.service_usage.extensions import kpi.schema_extensions.v2.tags.extensions import kpi.schema_extensions.v2.tos.extensions +import kpi.schema_extensions.v2.user_reports.extensions import kpi.schema_extensions.v2.users.extensions import kpi.schema_extensions.v2.versions.extensions diff --git a/kpi/schema_extensions/v2/user_reports/extensions.py b/kpi/schema_extensions/v2/user_reports/extensions.py new file mode 100644 index 0000000000..199b31c1b0 --- /dev/null +++ b/kpi/schema_extensions/v2/user_reports/extensions.py @@ -0,0 +1,193 @@ +from drf_spectacular.extensions import OpenApiSerializerFieldExtension +from drf_spectacular.plumbing import ( + build_array_type, + build_basic_type, + build_object_type, +) +from drf_spectacular.types import OpenApiTypes + +from kpi.schema_extensions.v2.generic.schema import ( + GENERIC_OBJECT_SCHEMA, + GENERIC_STRING_SCHEMA, +) + + +class OrganizationsFieldExtensions(OpenApiSerializerFieldExtension): + target_class = 'kpi.schema_extensions.v2.user_reports.fields.OrganizationsField' + + def map_serializer_field(self, auto_schema, direction): + return build_object_type( + properties={ + 'organization_name': GENERIC_STRING_SCHEMA, + 'organization_uid': GENERIC_STRING_SCHEMA, + 'role': GENERIC_STRING_SCHEMA, + } + ) + + +SUBSCRIPTION_METADATA_SCHEMA = build_object_type( + properties={ + 'request_url': GENERIC_STRING_SCHEMA, + 'organization_id': GENERIC_STRING_SCHEMA, + 'kpi_owner_user_id': GENERIC_STRING_SCHEMA, + 'kpi_owner_username': GENERIC_STRING_SCHEMA, + } +) + +PRICE_RECURRING_SCHEMA = build_object_type( + properties={ + 'meter': GENERIC_STRING_SCHEMA, + 'interval': GENERIC_STRING_SCHEMA, + 'usage_type': GENERIC_STRING_SCHEMA, + 'interval_count': build_basic_type(OpenApiTypes.INT), + 'aggregate_usage': GENERIC_STRING_SCHEMA, + 'trial_period_days': build_basic_type(OpenApiTypes.INT), + } +) + +TRANSFORM_QUANTITY_SCHEMA = build_object_type( + properties={ + 'divide_by': build_basic_type(OpenApiTypes.INT), + 'round': GENERIC_STRING_SCHEMA, + }, + nullable=True, +) + +PRODUCT_SCHEMA = build_object_type( + properties={ + 'id': GENERIC_STRING_SCHEMA, + 'name': GENERIC_STRING_SCHEMA, + 'description': GENERIC_STRING_SCHEMA, + 'type': GENERIC_STRING_SCHEMA, + 'metadata': build_object_type( + properties={ + 'product_type': GENERIC_STRING_SCHEMA, + 'storage_bytes_limit': GENERIC_STRING_SCHEMA, + }, + additionalProperties=True, + ), + } +) + +PRICE_SCHEMA = build_object_type( + properties={ + 'id': GENERIC_STRING_SCHEMA, + 'nickname': GENERIC_STRING_SCHEMA, + 'currency': GENERIC_STRING_SCHEMA, + 'type': GENERIC_STRING_SCHEMA, + 'recurring': PRICE_RECURRING_SCHEMA, + 'unit_amount': build_basic_type(OpenApiTypes.INT), + 'human_readable_price': GENERIC_STRING_SCHEMA, + 'metadata': build_object_type(additionalProperties=True), + 'active': build_basic_type(OpenApiTypes.BOOL), + 'product': PRODUCT_SCHEMA, + 'transform_quantity': TRANSFORM_QUANTITY_SCHEMA, + } +) + +SUBSCRIPTION_ITEM_SCHEMA = build_object_type( + properties={ + 'id': GENERIC_STRING_SCHEMA, + 'price': build_array_type(schema=PRICE_SCHEMA), + 'quantity': build_basic_type(OpenApiTypes.INT), + } +) + +SUBSCRIPTION_PHASE_ITEM_SCHEMA = build_object_type( + properties={ + 'plan': GENERIC_STRING_SCHEMA, + 'price': GENERIC_STRING_SCHEMA, + 'metadata': build_object_type(additionalProperties=True), + 'quantity': build_basic_type(OpenApiTypes.INT), + 'tax_rates': build_array_type(schema=build_basic_type(OpenApiTypes.INT)), + 'billing_thresholds': build_object_type(additionalProperties=True), + } +) + +SUBSCRIPTION_PHASE_SCHEMA = build_object_type( + properties={ + 'items': build_array_type(schema=SUBSCRIPTION_PHASE_ITEM_SCHEMA), + 'coupon': GENERIC_STRING_SCHEMA, + 'currency': GENERIC_STRING_SCHEMA, + 'end_date': build_basic_type(OpenApiTypes.INT), + 'metadata': build_object_type(), + 'trial_end': build_basic_type(OpenApiTypes.INT), + 'start_date': build_basic_type(OpenApiTypes.INT), + 'description': GENERIC_STRING_SCHEMA, + 'on_behalf_of': GENERIC_STRING_SCHEMA, + 'automatic_tax': build_object_type( + schema={'enabled': build_basic_type(OpenApiTypes.BOOL)} + ), + 'transfer_data': GENERIC_STRING_SCHEMA, + 'invoice_settings': GENERIC_STRING_SCHEMA, + 'add_invoice_items': GENERIC_OBJECT_SCHEMA, + 'collection_method': GENERIC_STRING_SCHEMA, + 'default_tax_rates': GENERIC_OBJECT_SCHEMA, + 'billing_thresholds': GENERIC_STRING_SCHEMA, + 'proration_behavior': GENERIC_STRING_SCHEMA, + 'billing_cycle_anchor': build_basic_type(OpenApiTypes.INT), + 'default_payment_method': GENERIC_STRING_SCHEMA, + 'application_fee_percent': build_basic_type(OpenApiTypes.INT), + }, + nullable=True, +) + +SUBSCRIPTION_SCHEDULE_SCHEMA = build_object_type( + properties={ + 'phases': build_array_type(schema=SUBSCRIPTION_PHASE_SCHEMA), + 'status': GENERIC_STRING_SCHEMA, + }, + nullable=True, +) + +SUBSCRIPTION_SCHEMA = build_object_type( + properties={ + 'items': build_array_type(schema=SUBSCRIPTION_ITEM_SCHEMA), + 'schedule': SUBSCRIPTION_SCHEDULE_SCHEMA, + 'djstripe_created': GENERIC_STRING_SCHEMA, + 'djstripe_updated': GENERIC_STRING_SCHEMA, + 'id': GENERIC_STRING_SCHEMA, + 'livemode': build_basic_type(OpenApiTypes.BOOL), + 'created': GENERIC_STRING_SCHEMA, + 'metadata': SUBSCRIPTION_METADATA_SCHEMA, + 'description': GENERIC_STRING_SCHEMA, + 'application_fee_percent': build_basic_type(OpenApiTypes.FLOAT), + 'billing_cycle_anchor': GENERIC_STRING_SCHEMA, + 'billing_thresholds': GENERIC_STRING_SCHEMA, + 'cancel_at': GENERIC_STRING_SCHEMA, + 'cancel_at_period_end': build_basic_type(OpenApiTypes.BOOL), + 'canceled_at': GENERIC_STRING_SCHEMA, + 'collection_method': GENERIC_STRING_SCHEMA, + 'current_period_end': GENERIC_STRING_SCHEMA, + 'current_period_start': GENERIC_STRING_SCHEMA, + 'days_until_due': GENERIC_STRING_SCHEMA, + 'discount': GENERIC_STRING_SCHEMA, + 'ended_at': GENERIC_STRING_SCHEMA, + 'next_pending_invoice_item_invoice': GENERIC_STRING_SCHEMA, + 'pause_collection': GENERIC_STRING_SCHEMA, + 'pending_invoice_item_interval': GENERIC_STRING_SCHEMA, + 'pending_update': GENERIC_STRING_SCHEMA, + 'proration_behavior': GENERIC_STRING_SCHEMA, + 'proration_date': GENERIC_STRING_SCHEMA, + 'quantity': build_basic_type(OpenApiTypes.INT), + 'start_date': GENERIC_STRING_SCHEMA, + 'status': GENERIC_STRING_SCHEMA, + 'trial_end': GENERIC_STRING_SCHEMA, + 'trial_start': GENERIC_STRING_SCHEMA, + 'djstripe_owner_account': GENERIC_STRING_SCHEMA, + 'customer': GENERIC_STRING_SCHEMA, + 'default_payment_method': GENERIC_STRING_SCHEMA, + 'default_source': GENERIC_STRING_SCHEMA, + 'latest_invoice': GENERIC_STRING_SCHEMA, + 'pending_setup_intent': GENERIC_STRING_SCHEMA, + 'plan': build_basic_type(OpenApiTypes.INT), + 'default_tax_rates': GENERIC_OBJECT_SCHEMA, + } +) + + +class SubscriptionsFieldExtension(OpenApiSerializerFieldExtension): + target_class = 'kpi.schema_extensions.v2.user_reports.fields.SubscriptionsField' + + def map_serializer_field(self, auto_schema, direction): + return build_array_type(schema=SUBSCRIPTION_SCHEMA) diff --git a/kpi/schema_extensions/v2/user_reports/fields.py b/kpi/schema_extensions/v2/user_reports/fields.py new file mode 100644 index 0000000000..c7d0444476 --- /dev/null +++ b/kpi/schema_extensions/v2/user_reports/fields.py @@ -0,0 +1,9 @@ +from rest_framework import serializers + + +class OrganizationsField(serializers.JSONField): + pass + + +class SubscriptionsField(serializers.JSONField): + pass diff --git a/kpi/schema_extensions/v2/user_reports/serializers.py b/kpi/schema_extensions/v2/user_reports/serializers.py new file mode 100644 index 0000000000..c78434ae43 --- /dev/null +++ b/kpi/schema_extensions/v2/user_reports/serializers.py @@ -0,0 +1,39 @@ +from rest_framework import serializers + +from kpi.schema_extensions.v2.me.fields import SocialAccountField +from kpi.schema_extensions.v2.service_usage.serializers import ServiceUsageResponse +from kpi.schema_extensions.v2.user_reports.fields import ( + OrganizationsField, + SubscriptionsField, +) +from kpi.schema_extensions.v2.users.fields import MetadataField +from kpi.utils.schema_extensions.serializers import inline_serializer_class + +UserReportsListResponse = inline_serializer_class( + name='UserReportsListResponse', + fields={ + 'extra_details_uid': serializers.CharField(), + 'username': serializers.CharField(), + 'first_name': serializers.CharField(), + 'last_name': serializers.CharField(), + 'email': serializers.EmailField(), + 'is_superuser': serializers.BooleanField(), + 'is_staff': serializers.BooleanField(), + 'is_active': serializers.BooleanField(), + 'date_joined': serializers.DateTimeField(), + 'last_login': serializers.DateTimeField(), + 'validated_email': serializers.BooleanField(), + 'validated_password': serializers.BooleanField(), + 'mfa_is_active': serializers.BooleanField(), + 'sso_is_active': serializers.BooleanField(), + 'accepted_tos': serializers.BooleanField(), + 'social_accounts': SocialAccountField(), + 'organizations': OrganizationsField(), + 'metadata': MetadataField(), + 'subscriptions': SubscriptionsField(), + 'current_service_usage': ServiceUsageResponse(), + 'account_restricted': serializers.BooleanField(), + 'asset_count': serializers.IntegerField(), + 'deployed_asset_count': serializers.IntegerField(), + }, +) diff --git a/kpi/tests/test_query_parser.py b/kpi/tests/test_query_parser.py new file mode 100644 index 0000000000..a60f3f5e69 --- /dev/null +++ b/kpi/tests/test_query_parser.py @@ -0,0 +1,63 @@ +from django.test import TestCase + +from kpi.utils.query_parser.query_parser import QueryParseActions + + +class TestQueryParseActionsProcessValue(TestCase): + def setUp(self): + self.query_parse_actions = QueryParseActions([], 3) + + def test_boolean_values(self): + # Test various boolean inputs + test_cases = [ + ('true', True), + ('True', True), + (True, True), + ('false', False), + ('FALSE', False), + (False, False), + ] + + for input_value, expected in test_cases: + with self.subTest(input_value=input_value): + result = self.query_parse_actions.process_value('field', input_value) + self.assertEqual(result, expected) + self.assertIsInstance(result, bool) + + def test_numeric_values(self): + # Test integer values + test_cases = [ + ('42', 42), + (42, 42), + ('-17', -17), + # Test float values + ('3.14', 3.14), + (3.14, 3.14), + ('-2.5', -2.5), + ] + + for input_value, expected in test_cases: + with self.subTest(input_value=input_value): + result = self.query_parse_actions.process_value('field', input_value) + self.assertEqual(result, expected) + self.assertIsInstance(result, type(expected)) + + def test_string_values(self): + # Test string values + test_cases = [ + 'hello', + 'Hello World', + '123abc', + 'special!@#$', + ] + + for input_value in test_cases: + with self.subTest(input_value=input_value): + result = self.query_parse_actions.process_value('field', input_value) + self.assertEqual(result, input_value) + self.assertIsInstance(result, str) + + def test_null_value(self): + # Test null value + result = self.query_parse_actions.process_value('field', 'null') + self.assertIsNone(result) diff --git a/kpi/urls/router_api_v2.py b/kpi/urls/router_api_v2.py index b773a67439..61246a14c5 100644 --- a/kpi/urls/router_api_v2.py +++ b/kpi/urls/router_api_v2.py @@ -15,6 +15,7 @@ from kobo.apps.project_ownership.urls import router as project_ownership_router from kobo.apps.project_views.views import ProjectViewViewSet from kpi.renderers import BasicHTMLRenderer +from kobo.apps.user_reports.views import UserReportsViewSet from kpi.views.v2.asset import AssetViewSet from kpi.views.v2.asset_counts import AssetCountsViewSet from kpi.views.v2.asset_export_settings import AssetExportSettingsViewSet @@ -186,7 +187,7 @@ def get_urls(self, *args, **kwargs): router_api_v2.register(r'service_usage', ServiceUsageViewSet, basename='service-usage') router_api_v2.register(r'users', UserViewSet, basename='user-kpi') - +router_api_v2.register(r'user-reports', UserReportsViewSet, basename='user-reports') router_api_v2.register(r'tags', TagViewSet, basename='tags') router_api_v2.register( r'terms-of-service', TermsOfServiceViewSet, basename='terms-of-service' diff --git a/kpi/utils/query_parser/query_parser.py b/kpi/utils/query_parser/query_parser.py index d06e57bbb7..6dd9e7589d 100644 --- a/kpi/utils/query_parser/query_parser.py +++ b/kpi/utils/query_parser/query_parser.py @@ -1,4 +1,3 @@ -# coding: utf-8 import operator import re from collections import defaultdict @@ -56,8 +55,8 @@ def __init__(self, default_field_lookups: list, min_search_characters: int): self.min_search_characters = min_search_characters self.has_term_with_sufficient_length = False - @staticmethod - def process_value(field, value): + @classmethod + def process_value(cls, field, value): # If all we're doing when we have a type mismatch with a field # is returning an empty set, then we don't need to do type validation. # Django compares between field values and string versions just fine. @@ -80,6 +79,8 @@ def process_value(field, value): if lower_value in ['true', 'false']: return bool(util.strtobool(lower_value)) + value = cls._normalize_numeric_value(value) + return value @staticmethod @@ -259,6 +260,27 @@ def string(text, a, b, elements): def name(text, a, b, elements): return text[a:b] + @staticmethod + def _normalize_numeric_value(value): + + if isinstance(value, (int, float)): + return value + + if isinstance(value, str): + # Try `int` first + try: + return int(value) + except ValueError: + pass + + # Try `float` after + try: + return float(value) + except ValueError: + pass + + return value + def get_parsed_parameters(parsed_query: Q) -> dict: """ diff --git a/static/openapi/schema_openrosa.json b/static/openapi/schema_openrosa.json index a9bf5680e0..d4cb421b72 100644 --- a/static/openapi/schema_openrosa.json +++ b/static/openapi/schema_openrosa.json @@ -1040,4 +1040,4 @@ } } } -} \ No newline at end of file +} diff --git a/static/openapi/schema_v2.json b/static/openapi/schema_v2.json index ff08d4cc0c..e1d64f626d 100644 --- a/static/openapi/schema_v2.json +++ b/static/openapi/schema_v2.json @@ -38,7 +38,7 @@ } ], "tags": [ - "Audit logs (superusers)" + "Server logs (superusers)" ], "security": [ { @@ -85,7 +85,7 @@ "operationId": "api_v2_access_logs_export_list", "description": "## List all access logs export tasks for all users\n\n⚠️ _Only available to superusers_\n", "tags": [ - "Audit logs (superusers)" + "Server logs (superusers)" ], "security": [ { @@ -133,7 +133,7 @@ "operationId": "api_v2_access_logs_export_create", "description": "## Create an export task for all users\n\n⚠️ _Only available to superusers_\n", "tags": [ - "Audit logs (superusers)" + "Server logs (superusers)" ], "security": [ { @@ -7611,7 +7611,7 @@ } ], "tags": [ - "Audit logs (superusers)" + "Server logs (superusers)" ], "security": [ { @@ -9329,7 +9329,7 @@ } ], "tags": [ - "Audit logs (superusers)" + "Server logs (superusers)" ], "security": [ { @@ -9394,7 +9394,7 @@ "operationId": "api_v2_project_history_logs_export_retrieve", "description": "## List of Project History Exports\n\n⚠️ _Only available to superusers_\n", "tags": [ - "Audit logs (superusers)" + "Server logs (superusers)" ], "security": [ { @@ -9457,7 +9457,7 @@ "operationId": "api_v2_project_history_logs_export_create", "description": "## Create an export of projects history logs\n\n⚠️ _Only available to superusers_\n\nExport project history logs and send it by email to the requesting user.\n", "tags": [ - "Audit logs (superusers)" + "Server logs (superusers)" ], "security": [ { @@ -11644,6 +11644,280 @@ } } }, + "/api/v2/user-reports/": { + "get": { + "operationId": "api_v2_user_reports_list", + "description": "## List user reports\n", + "parameters": [ + { + "in": "query", + "name": "current_period_submissions_max", + "schema": { + "type": "integer", + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "format": "int64" + } + }, + { + "in": "query", + "name": "current_period_submissions_min", + "schema": { + "type": "integer", + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "format": "int64" + } + }, + { + "in": "query", + "name": "date_joined_after", + "schema": { + "type": "string", + "format": "date-time" + } + }, + { + "in": "query", + "name": "date_joined_before", + "schema": { + "type": "string", + "format": "date-time" + } + }, + { + "in": "query", + "name": "email", + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "has_subscriptions", + "schema": { + "type": "boolean" + } + }, + { + "in": "query", + "name": "last_login_after", + "schema": { + "type": "string", + "format": "date-time" + } + }, + { + "in": "query", + "name": "last_login_before", + "schema": { + "type": "string", + "format": "date-time" + } + }, + { + "name": "limit", + "required": false, + "in": "query", + "description": "Number of results to return per page.", + "schema": { + "type": "integer" + } + }, + { + "in": "query", + "name": "nlp_usage_asr_seconds_total_max", + "schema": { + "type": "integer", + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "format": "int64" + } + }, + { + "in": "query", + "name": "nlp_usage_asr_seconds_total_min", + "schema": { + "type": "integer", + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "format": "int64" + } + }, + { + "in": "query", + "name": "nlp_usage_mt_characters_total_max", + "schema": { + "type": "integer", + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "format": "int64" + } + }, + { + "in": "query", + "name": "nlp_usage_mt_characters_total_min", + "schema": { + "type": "integer", + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "format": "int64" + } + }, + { + "name": "offset", + "required": false, + "in": "query", + "description": "The initial index from which to return the results.", + "schema": { + "type": "integer" + } + }, + { + "name": "ordering", + "required": false, + "in": "query", + "description": "Which field to use when ordering the results.", + "schema": { + "type": "string" + } + }, + { + "name": "search", + "required": false, + "in": "query", + "description": "A search term.", + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "storage_bytes_total_max", + "schema": { + "type": "integer", + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "format": "int64" + } + }, + { + "in": "query", + "name": "storage_bytes_total_min", + "schema": { + "type": "integer", + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "format": "int64" + } + }, + { + "in": "query", + "name": "submission_counts_all_time_max", + "schema": { + "type": "integer", + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "format": "int64" + } + }, + { + "in": "query", + "name": "submission_counts_all_time_min", + "schema": { + "type": "integer", + "maximum": 9223372036854775807, + "minimum": -9223372036854775808, + "format": "int64" + } + }, + { + "in": "query", + "name": "subscription_id", + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "subscription_status", + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "username", + "schema": { + "type": "string" + } + } + ], + "tags": [ + "User Reports" + ], + "security": [ + { + "BasicAuth": [] + }, + { + "TokenAuth": [] + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PaginatedUserReportsListResponseList" + } + } + }, + "description": null + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorDetail" + }, + "examples": { + "AccessDenied": { + "value": { + "detail": "You do not have permission to perform this action." + }, + "summary": "Access Denied" + } + } + } + }, + "description": "" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorObject" + }, + "examples": { + "BadRequest": { + "value": { + "detail": { + "field_name": [ + "Error message" + ] + } + }, + "summary": "Bad request" + } + } + } + }, + "description": "" + } + } + } + }, "/api/v2/users/": { "get": { "operationId": "api_v2_users_list", @@ -17597,6 +17871,37 @@ } } }, + "PaginatedUserReportsListResponseList": { + "type": "object", + "required": [ + "count", + "results" + ], + "properties": { + "count": { + "type": "integer", + "example": 123 + }, + "next": { + "type": "string", + "nullable": true, + "format": "uri", + "example": "http://api.example.org/accounts/?offset=400&limit=100" + }, + "previous": { + "type": "string", + "nullable": true, + "format": "uri", + "example": "http://api.example.org/accounts/?offset=200&limit=100" + }, + "results": { + "type": "array", + "items": { + "$ref": "#/components/schemas/UserReportsListResponse" + } + } + } + }, "PaginatedVersionListResponseList": { "type": "object", "required": [ @@ -20096,6 +20401,549 @@ "username" ] }, + "UserReportsListResponse": { + "type": "object", + "properties": { + "extra_details_uid": { + "type": "string" + }, + "username": { + "type": "string" + }, + "first_name": { + "type": "string" + }, + "last_name": { + "type": "string" + }, + "email": { + "type": "string", + "format": "email" + }, + "is_superuser": { + "type": "boolean" + }, + "is_staff": { + "type": "boolean" + }, + "is_active": { + "type": "boolean" + }, + "date_joined": { + "type": "string", + "format": "date-time" + }, + "last_login": { + "type": "string", + "format": "date-time" + }, + "validated_email": { + "type": "boolean" + }, + "validated_password": { + "type": "boolean" + }, + "mfa_is_active": { + "type": "boolean" + }, + "sso_is_active": { + "type": "boolean" + }, + "accepted_tos": { + "type": "boolean" + }, + "social_accounts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { + "type": "string" + }, + "uid": { + "type": "string" + }, + "last_joined": { + "type": "string", + "format": "date-time" + }, + "date_joined": { + "type": "string", + "format": "date-time" + }, + "email": { + "type": "string", + "format": "email" + }, + "username": { + "type": "string" + } + } + } + }, + "organizations": { + "type": "object", + "properties": { + "organization_name": { + "type": "string" + }, + "organization_uid": { + "type": "string" + }, + "role": { + "type": "string" + } + } + }, + "metadata": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "sector": { + "type": "string" + }, + "country": { + "type": "string" + }, + "organization": { + "type": "string" + }, + "last_ui_language": { + "type": "string" + }, + "organization_type": { + "type": "string" + }, + "organization_website": { + "type": "string" + }, + "project_views_settings": { + "type": "object", + "properties": { + "kobo_my_project": { + "type": "object", + "properties": { + "order": { + "type": "object" + }, + "fields": { + "type": "array", + "items": { + "type": "string" + } + }, + "filters": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + }, + "subscriptions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "price": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "nickname": { + "type": "string" + }, + "currency": { + "type": "string" + }, + "type": { + "type": "string" + }, + "recurring": { + "type": "object", + "properties": { + "meter": { + "type": "string" + }, + "interval": { + "type": "string" + }, + "usage_type": { + "type": "string" + }, + "interval_count": { + "type": "integer" + }, + "aggregate_usage": { + "type": "string" + }, + "trial_period_days": { + "type": "integer" + } + } + }, + "unit_amount": { + "type": "integer" + }, + "human_readable_price": { + "type": "string" + }, + "metadata": { + "type": "object", + "additionalProperties": true + }, + "active": { + "type": "boolean" + }, + "product": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "type": { + "type": "string" + }, + "metadata": { + "type": "object", + "properties": { + "product_type": { + "type": "string" + }, + "storage_bytes_limit": { + "type": "string" + } + }, + "additionalProperties": true + } + } + }, + "transform_quantity": { + "type": "object", + "properties": { + "divide_by": { + "type": "integer" + }, + "round": { + "type": "string" + } + }, + "nullable": true + } + } + } + }, + "quantity": { + "type": "integer" + } + } + } + }, + "schedule": { + "type": "object", + "properties": { + "phases": { + "type": "array", + "items": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "plan": { + "type": "string" + }, + "price": { + "type": "string" + }, + "metadata": { + "type": "object", + "additionalProperties": true + }, + "quantity": { + "type": "integer" + }, + "tax_rates": { + "type": "array", + "items": { + "type": "integer" + } + }, + "billing_thresholds": { + "type": "object", + "additionalProperties": true + } + } + } + }, + "coupon": { + "type": "string" + }, + "currency": { + "type": "string" + }, + "end_date": { + "type": "integer" + }, + "metadata": { + "type": "object" + }, + "trial_end": { + "type": "integer" + }, + "start_date": { + "type": "integer" + }, + "description": { + "type": "string" + }, + "on_behalf_of": { + "type": "string" + }, + "automatic_tax": { + "type": "object", + "schema": { + "enabled": { + "type": "boolean" + } + } + }, + "transfer_data": { + "type": "string" + }, + "invoice_settings": { + "type": "string" + }, + "add_invoice_items": { + "type": "object" + }, + "collection_method": { + "type": "string" + }, + "default_tax_rates": { + "type": "object" + }, + "billing_thresholds": { + "type": "string" + }, + "proration_behavior": { + "type": "string" + }, + "billing_cycle_anchor": { + "type": "integer" + }, + "default_payment_method": { + "type": "string" + }, + "application_fee_percent": { + "type": "integer" + } + }, + "nullable": true + } + }, + "status": { + "type": "string" + } + }, + "nullable": true + }, + "djstripe_created": { + "type": "string" + }, + "djstripe_updated": { + "type": "string" + }, + "id": { + "type": "string" + }, + "livemode": { + "type": "boolean" + }, + "created": { + "type": "string" + }, + "metadata": { + "type": "object", + "properties": { + "request_url": { + "type": "string" + }, + "organization_id": { + "type": "string" + }, + "kpi_owner_user_id": { + "type": "string" + }, + "kpi_owner_username": { + "type": "string" + } + } + }, + "description": { + "type": "string" + }, + "application_fee_percent": { + "type": "number", + "format": "float" + }, + "billing_cycle_anchor": { + "type": "string" + }, + "billing_thresholds": { + "type": "string" + }, + "cancel_at": { + "type": "string" + }, + "cancel_at_period_end": { + "type": "boolean" + }, + "canceled_at": { + "type": "string" + }, + "collection_method": { + "type": "string" + }, + "current_period_end": { + "type": "string" + }, + "current_period_start": { + "type": "string" + }, + "days_until_due": { + "type": "string" + }, + "discount": { + "type": "string" + }, + "ended_at": { + "type": "string" + }, + "next_pending_invoice_item_invoice": { + "type": "string" + }, + "pause_collection": { + "type": "string" + }, + "pending_invoice_item_interval": { + "type": "string" + }, + "pending_update": { + "type": "string" + }, + "proration_behavior": { + "type": "string" + }, + "proration_date": { + "type": "string" + }, + "quantity": { + "type": "integer" + }, + "start_date": { + "type": "string" + }, + "status": { + "type": "string" + }, + "trial_end": { + "type": "string" + }, + "trial_start": { + "type": "string" + }, + "djstripe_owner_account": { + "type": "string" + }, + "customer": { + "type": "string" + }, + "default_payment_method": { + "type": "string" + }, + "default_source": { + "type": "string" + }, + "latest_invoice": { + "type": "string" + }, + "pending_setup_intent": { + "type": "string" + }, + "plan": { + "type": "integer" + }, + "default_tax_rates": { + "type": "object" + } + } + } + }, + "current_service_usage": { + "$ref": "#/components/schemas/ServiceUsageResponse" + }, + "account_restricted": { + "type": "boolean" + }, + "asset_count": { + "type": "integer" + }, + "deployed_asset_count": { + "type": "integer" + } + }, + "required": [ + "accepted_tos", + "account_restricted", + "asset_count", + "current_service_usage", + "date_joined", + "deployed_asset_count", + "email", + "extra_details_uid", + "first_name", + "is_active", + "is_staff", + "is_superuser", + "last_login", + "last_name", + "metadata", + "mfa_is_active", + "organizations", + "social_accounts", + "sso_is_active", + "subscriptions", + "username", + "validated_email", + "validated_password" + ] + }, "UserRetrieveResponse": { "type": "object", "properties": { @@ -20320,7 +21168,7 @@ "description": "Subscribe to and manage shared library collections" }, { - "name": "Audit logs (superusers)", + "name": "Server logs (superusers)", "description": "View server-wide logs" }, { @@ -20332,4 +21180,4 @@ "description": "Languages, available permissions, other" } ] -} \ No newline at end of file +} diff --git a/static/openapi/schema_v2.yaml b/static/openapi/schema_v2.yaml index da119e6f97..807cdefec9 100644 --- a/static/openapi/schema_v2.yaml +++ b/static/openapi/schema_v2.yaml @@ -56,7 +56,7 @@ paths: schema: type: string tags: - - Audit logs (superusers) + - Server logs (superusers) security: - BasicAuth: [] - TokenAuth: [] @@ -86,7 +86,7 @@ paths: ⚠️ _Only available to superusers_ tags: - - Audit logs (superusers) + - Server logs (superusers) security: - BasicAuth: [] - TokenAuth: [] @@ -117,7 +117,7 @@ paths: ⚠️ _Only available to superusers_ tags: - - Audit logs (superusers) + - Server logs (superusers) security: - BasicAuth: [] - TokenAuth: [] @@ -5601,7 +5601,7 @@ paths: schema: type: string tags: - - Audit logs (superusers) + - Server logs (superusers) security: - BasicAuth: [] - TokenAuth: [] @@ -6899,7 +6899,7 @@ paths: schema: type: string tags: - - Audit logs (superusers) + - Server logs (superusers) security: - BasicAuth: [] - TokenAuth: [] @@ -6940,7 +6940,7 @@ paths: ⚠️ _Only available to superusers_ tags: - - Audit logs (superusers) + - Server logs (superusers) security: - BasicAuth: [] - TokenAuth: [] @@ -6982,7 +6982,7 @@ paths: Export project history logs and send it by email to the requesting user. tags: - - Audit logs (superusers) + - Server logs (superusers) security: - BasicAuth: [] - TokenAuth: [] @@ -8421,6 +8421,182 @@ paths: detail: Not found. summary: Not Found description: '' + /api/v2/user-reports/: + get: + operationId: api_v2_user_reports_list + description: | + ## List user reports + parameters: + - in: query + name: current_period_submissions_max + schema: + type: integer + maximum: 9223372036854775807 + minimum: -9223372036854775808 + format: int64 + - in: query + name: current_period_submissions_min + schema: + type: integer + maximum: 9223372036854775807 + minimum: -9223372036854775808 + format: int64 + - in: query + name: date_joined_after + schema: + type: string + format: date-time + - in: query + name: date_joined_before + schema: + type: string + format: date-time + - in: query + name: email + schema: + type: string + - in: query + name: has_subscriptions + schema: + type: boolean + - in: query + name: last_login_after + schema: + type: string + format: date-time + - in: query + name: last_login_before + schema: + type: string + format: date-time + - name: limit + required: false + in: query + description: Number of results to return per page. + schema: + type: integer + - in: query + name: nlp_usage_asr_seconds_total_max + schema: + type: integer + maximum: 9223372036854775807 + minimum: -9223372036854775808 + format: int64 + - in: query + name: nlp_usage_asr_seconds_total_min + schema: + type: integer + maximum: 9223372036854775807 + minimum: -9223372036854775808 + format: int64 + - in: query + name: nlp_usage_mt_characters_total_max + schema: + type: integer + maximum: 9223372036854775807 + minimum: -9223372036854775808 + format: int64 + - in: query + name: nlp_usage_mt_characters_total_min + schema: + type: integer + maximum: 9223372036854775807 + minimum: -9223372036854775808 + format: int64 + - name: offset + required: false + in: query + description: The initial index from which to return the results. + schema: + type: integer + - name: ordering + required: false + in: query + description: Which field to use when ordering the results. + schema: + type: string + - name: search + required: false + in: query + description: A search term. + schema: + type: string + - in: query + name: storage_bytes_total_max + schema: + type: integer + maximum: 9223372036854775807 + minimum: -9223372036854775808 + format: int64 + - in: query + name: storage_bytes_total_min + schema: + type: integer + maximum: 9223372036854775807 + minimum: -9223372036854775808 + format: int64 + - in: query + name: submission_counts_all_time_max + schema: + type: integer + maximum: 9223372036854775807 + minimum: -9223372036854775808 + format: int64 + - in: query + name: submission_counts_all_time_min + schema: + type: integer + maximum: 9223372036854775807 + minimum: -9223372036854775808 + format: int64 + - in: query + name: subscription_id + schema: + type: string + - in: query + name: subscription_status + schema: + type: string + - in: query + name: username + schema: + type: string + tags: + - User Reports + security: + - BasicAuth: [] + - TokenAuth: [] + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/PaginatedUserReportsListResponseList' + description: null + '403': + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorDetail' + examples: + AccessDenied: + value: + detail: You do not have permission to perform this action. + summary: Access Denied + description: '' + '400': + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorObject' + examples: + BadRequest: + value: + detail: + field_name: + - Error message + summary: Bad request + description: '' /api/v2/users/: get: operationId: api_v2_users_list @@ -12623,6 +12799,29 @@ components: type: array items: $ref: '#/components/schemas/UserListResponse' + PaginatedUserReportsListResponseList: + type: object + required: + - count + - results + properties: + count: + type: integer + example: 123 + next: + type: string + nullable: true + format: uri + example: http://api.example.org/accounts/?offset=400&limit=100 + previous: + type: string + nullable: true + format: uri + example: http://api.example.org/accounts/?offset=200&limit=100 + results: + type: array + items: + $ref: '#/components/schemas/UserReportsListResponse' PaginatedVersionListResponseList: type: object required: @@ -14510,6 +14709,374 @@ components: - last_login - metadata - username + UserReportsListResponse: + type: object + properties: + extra_details_uid: + type: string + username: + type: string + first_name: + type: string + last_name: + type: string + email: + type: string + format: email + is_superuser: + type: boolean + is_staff: + type: boolean + is_active: + type: boolean + date_joined: + type: string + format: date-time + last_login: + type: string + format: date-time + validated_email: + type: boolean + validated_password: + type: boolean + mfa_is_active: + type: boolean + sso_is_active: + type: boolean + accepted_tos: + type: boolean + social_accounts: + type: array + items: + type: object + properties: + provider: + type: string + uid: + type: string + last_joined: + type: string + format: date-time + date_joined: + type: string + format: date-time + email: + type: string + format: email + username: + type: string + organizations: + type: object + properties: + organization_name: + type: string + organization_uid: + type: string + role: + type: string + metadata: + type: object + properties: + name: + type: string + sector: + type: string + country: + type: string + organization: + type: string + last_ui_language: + type: string + organization_type: + type: string + organization_website: + type: string + project_views_settings: + type: object + properties: + kobo_my_project: + type: object + properties: + order: + type: object + fields: + type: array + items: + type: string + filters: + type: array + items: + type: string + subscriptions: + type: array + items: + type: object + properties: + items: + type: array + items: + type: object + properties: + id: + type: string + price: + type: array + items: + type: object + properties: + id: + type: string + nickname: + type: string + currency: + type: string + type: + type: string + recurring: + type: object + properties: + meter: + type: string + interval: + type: string + usage_type: + type: string + interval_count: + type: integer + aggregate_usage: + type: string + trial_period_days: + type: integer + unit_amount: + type: integer + human_readable_price: + type: string + metadata: + type: object + additionalProperties: true + active: + type: boolean + product: + type: object + properties: + id: + type: string + name: + type: string + description: + type: string + type: + type: string + metadata: + type: object + properties: + product_type: + type: string + storage_bytes_limit: + type: string + additionalProperties: true + transform_quantity: + type: object + properties: + divide_by: + type: integer + round: + type: string + nullable: true + quantity: + type: integer + schedule: + type: object + properties: + phases: + type: array + items: + type: object + properties: + items: + type: array + items: + type: object + properties: + plan: + type: string + price: + type: string + metadata: + type: object + additionalProperties: true + quantity: + type: integer + tax_rates: + type: array + items: + type: integer + billing_thresholds: + type: object + additionalProperties: true + coupon: + type: string + currency: + type: string + end_date: + type: integer + metadata: + type: object + trial_end: + type: integer + start_date: + type: integer + description: + type: string + on_behalf_of: + type: string + automatic_tax: + type: object + schema: + enabled: + type: boolean + transfer_data: + type: string + invoice_settings: + type: string + add_invoice_items: + type: object + collection_method: + type: string + default_tax_rates: + type: object + billing_thresholds: + type: string + proration_behavior: + type: string + billing_cycle_anchor: + type: integer + default_payment_method: + type: string + application_fee_percent: + type: integer + nullable: true + status: + type: string + nullable: true + djstripe_created: + type: string + djstripe_updated: + type: string + id: + type: string + livemode: + type: boolean + created: + type: string + metadata: + type: object + properties: + request_url: + type: string + organization_id: + type: string + kpi_owner_user_id: + type: string + kpi_owner_username: + type: string + description: + type: string + application_fee_percent: + type: number + format: float + billing_cycle_anchor: + type: string + billing_thresholds: + type: string + cancel_at: + type: string + cancel_at_period_end: + type: boolean + canceled_at: + type: string + collection_method: + type: string + current_period_end: + type: string + current_period_start: + type: string + days_until_due: + type: string + discount: + type: string + ended_at: + type: string + next_pending_invoice_item_invoice: + type: string + pause_collection: + type: string + pending_invoice_item_interval: + type: string + pending_update: + type: string + proration_behavior: + type: string + proration_date: + type: string + quantity: + type: integer + start_date: + type: string + status: + type: string + trial_end: + type: string + trial_start: + type: string + djstripe_owner_account: + type: string + customer: + type: string + default_payment_method: + type: string + default_source: + type: string + latest_invoice: + type: string + pending_setup_intent: + type: string + plan: + type: integer + default_tax_rates: + type: object + current_service_usage: + $ref: '#/components/schemas/ServiceUsageResponse' + account_restricted: + type: boolean + asset_count: + type: integer + deployed_asset_count: + type: integer + required: + - accepted_tos + - account_restricted + - asset_count + - current_service_usage + - date_joined + - deployed_asset_count + - email + - extra_details_uid + - first_name + - is_active + - is_staff + - is_superuser + - last_login + - last_name + - metadata + - mfa_is_active + - organizations + - social_accounts + - sso_is_active + - subscriptions + - username + - validated_email + - validated_password UserRetrieveResponse: type: object properties: @@ -14665,7 +15232,7 @@ tags: description: Project history logs, access logs, Rest Service hook logs - name: Library collections description: Subscribe to and manage shared library collections -- name: Audit logs (superusers) +- name: Server logs (superusers) description: View server-wide logs - name: User / team / organization / usage description: Manage users, orgs, invites, roles, and usage tracking