Merge pull request 'Add comprehensive column check and migration 009' (#14) from dev into loaf-prod

Reviewed-on: #14

alembic/versions/009_add_all_missing_columns.py

@@ -0,0 +1,237 @@
+"""add_all_missing_columns
+
+Revision ID: 009_add_all_missing
+Revises: 008_add_donations
+Create Date: 2026-01-04
+
+Fixes:
+- Add ALL remaining missing columns across all tables
+- Users: newsletter preferences, volunteer, scholarship, directory, password reset, ToS, member_since, reminders, rejection, import tracking
+- Events: calendar_uid
+- Subscriptions: base_subscription_cents, donation_cents, manual_payment
+- ImportJobs: WordPress import fields
+- Create ImportRollbackAudit table if not exists
+"""
+from typing import Sequence, Union
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects.postgresql import UUID
+from sqlalchemy import inspect
+
+
+# revision identifiers, used by Alembic.
+revision: str = '009_add_all_missing'
+down_revision: Union[str, None] = '008_add_donations'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Add all missing columns across all tables"""
+
+    conn = op.get_bind()
+    inspector = inspect(conn)
+
+    # ============================================================
+    # 1. USERS TABLE - Add ~28 missing columns
+    # ============================================================
+    users_columns = {col['name'] for col in inspector.get_columns('users')}
+
+    # Newsletter publication preferences
+    if 'newsletter_publish_name' not in users_columns:
+        op.add_column('users', sa.Column('newsletter_publish_name', sa.Boolean(), nullable=False, server_default='false'))
+    if 'newsletter_publish_photo' not in users_columns:
+        op.add_column('users', sa.Column('newsletter_publish_photo', sa.Boolean(), nullable=False, server_default='false'))
+    if 'newsletter_publish_birthday' not in users_columns:
+        op.add_column('users', sa.Column('newsletter_publish_birthday', sa.Boolean(), nullable=False, server_default='false'))
+    if 'newsletter_publish_none' not in users_columns:
+        op.add_column('users', sa.Column('newsletter_publish_none', sa.Boolean(), nullable=False, server_default='false'))
+
+    # Volunteer interests
+    if 'volunteer_interests' not in users_columns:
+        op.add_column('users', sa.Column('volunteer_interests', sa.JSON(), nullable=True, server_default='[]'))
+
+    # Scholarship
+    if 'scholarship_requested' not in users_columns:
+        op.add_column('users', sa.Column('scholarship_requested', sa.Boolean(), nullable=False, server_default='false'))
+
+    # Directory
+    if 'show_in_directory' not in users_columns:
+        op.add_column('users', sa.Column('show_in_directory', sa.Boolean(), nullable=False, server_default='false'))
+
+    # Password reset
+    if 'password_reset_token' not in users_columns:
+        op.add_column('users', sa.Column('password_reset_token', sa.String(), nullable=True))
+    if 'password_reset_expires' not in users_columns:
+        op.add_column('users', sa.Column('password_reset_expires', sa.DateTime(), nullable=True))
+    if 'force_password_change' not in users_columns:
+        op.add_column('users', sa.Column('force_password_change', sa.Boolean(), nullable=False, server_default='false'))
+
+    # Terms of Service
+    if 'accepts_tos' not in users_columns:
+        op.add_column('users', sa.Column('accepts_tos', sa.Boolean(), nullable=False, server_default='false'))
+    if 'tos_accepted_at' not in users_columns:
+        op.add_column('users', sa.Column('tos_accepted_at', sa.DateTime(), nullable=True))
+
+    # Member since
+    if 'member_since' not in users_columns:
+        op.add_column('users', sa.Column('member_since', sa.DateTime(), nullable=True))
+
+    # Email verification reminders
+    if 'email_verification_reminders_sent' not in users_columns:
+        op.add_column('users', sa.Column('email_verification_reminders_sent', sa.Integer(), nullable=False, server_default='0'))
+    if 'last_email_verification_reminder_at' not in users_columns:
+        op.add_column('users', sa.Column('last_email_verification_reminder_at', sa.DateTime(), nullable=True))
+
+    # Event attendance reminders
+    if 'event_attendance_reminders_sent' not in users_columns:
+        op.add_column('users', sa.Column('event_attendance_reminders_sent', sa.Integer(), nullable=False, server_default='0'))
+    if 'last_event_attendance_reminder_at' not in users_columns:
+        op.add_column('users', sa.Column('last_event_attendance_reminder_at', sa.DateTime(), nullable=True))
+
+    # Payment reminders
+    if 'payment_reminders_sent' not in users_columns:
+        op.add_column('users', sa.Column('payment_reminders_sent', sa.Integer(), nullable=False, server_default='0'))
+    if 'last_payment_reminder_at' not in users_columns:
+        op.add_column('users', sa.Column('last_payment_reminder_at', sa.DateTime(), nullable=True))
+
+    # Renewal reminders
+    if 'renewal_reminders_sent' not in users_columns:
+        op.add_column('users', sa.Column('renewal_reminders_sent', sa.Integer(), nullable=False, server_default='0'))
+    if 'last_renewal_reminder_at' not in users_columns:
+        op.add_column('users', sa.Column('last_renewal_reminder_at', sa.DateTime(), nullable=True))
+
+    # Rejection tracking
+    if 'rejection_reason' not in users_columns:
+        op.add_column('users', sa.Column('rejection_reason', sa.Text(), nullable=True))
+    if 'rejected_at' not in users_columns:
+        op.add_column('users', sa.Column('rejected_at', sa.DateTime(timezone=True), nullable=True))
+    if 'rejected_by' not in users_columns:
+        op.add_column('users', sa.Column('rejected_by', UUID(as_uuid=True), nullable=True))
+        # Note: Foreign key constraint skipped to avoid circular dependency issues
+
+    # WordPress import tracking
+    if 'import_source' not in users_columns:
+        op.add_column('users', sa.Column('import_source', sa.String(50), nullable=True))
+    if 'import_job_id' not in users_columns:
+        op.add_column('users', sa.Column('import_job_id', UUID(as_uuid=True), nullable=True))
+        # Note: Foreign key will be added after import_jobs table is updated
+    if 'wordpress_user_id' not in users_columns:
+        op.add_column('users', sa.Column('wordpress_user_id', sa.BigInteger(), nullable=True))
+    if 'wordpress_registered_date' not in users_columns:
+        op.add_column('users', sa.Column('wordpress_registered_date', sa.DateTime(timezone=True), nullable=True))
+
+    # ============================================================
+    # 2. EVENTS TABLE - Add calendar_uid
+    # ============================================================
+    events_columns = {col['name'] for col in inspector.get_columns('events')}
+
+    if 'calendar_uid' not in events_columns:
+        op.add_column('events', sa.Column('calendar_uid', sa.String(), nullable=True))
+
+    # ============================================================
+    # 3. SUBSCRIPTIONS TABLE - Add donation tracking
+    # ============================================================
+    subscriptions_columns = {col['name'] for col in inspector.get_columns('subscriptions')}
+
+    if 'base_subscription_cents' not in subscriptions_columns:
+        op.add_column('subscriptions', sa.Column('base_subscription_cents', sa.Integer(), nullable=True))
+        # Update existing rows: base_subscription_cents = amount_paid_cents - donation_cents (default 0)
+        op.execute("UPDATE subscriptions SET base_subscription_cents = COALESCE(amount_paid_cents, 0) WHERE base_subscription_cents IS NULL")
+        # Make it non-nullable after populating
+        op.alter_column('subscriptions', 'base_subscription_cents', nullable=False)
+
+    if 'donation_cents' not in subscriptions_columns:
+        op.add_column('subscriptions', sa.Column('donation_cents', sa.Integer(), nullable=False, server_default='0'))
+
+    if 'manual_payment' not in subscriptions_columns:
+        op.add_column('subscriptions', sa.Column('manual_payment', sa.Boolean(), nullable=False, server_default='false'))
+
+    # ============================================================
+    # 4. IMPORT_JOBS TABLE - Add WordPress import fields
+    # ============================================================
+    import_jobs_columns = {col['name'] for col in inspector.get_columns('import_jobs')}
+
+    if 'field_mapping' not in import_jobs_columns:
+        op.add_column('import_jobs', sa.Column('field_mapping', sa.JSON(), nullable=False, server_default='{}'))
+
+    if 'wordpress_metadata' not in import_jobs_columns:
+        op.add_column('import_jobs', sa.Column('wordpress_metadata', sa.JSON(), nullable=False, server_default='{}'))
+
+    if 'imported_user_ids' not in import_jobs_columns:
+        op.add_column('import_jobs', sa.Column('imported_user_ids', sa.JSON(), nullable=False, server_default='[]'))
+
+    if 'rollback_at' not in import_jobs_columns:
+        op.add_column('import_jobs', sa.Column('rollback_at', sa.DateTime(), nullable=True))
+
+    if 'rollback_by' not in import_jobs_columns:
+        op.add_column('import_jobs', sa.Column('rollback_by', UUID(as_uuid=True), nullable=True))
+        # Foreign key will be added if needed
+
+    # ============================================================
+    # 5. CREATE IMPORT_ROLLBACK_AUDIT TABLE
+    # ============================================================
+    if 'import_rollback_audit' not in inspector.get_table_names():
+        op.create_table(
+            'import_rollback_audit',
+            sa.Column('id', UUID(as_uuid=True), primary_key=True),
+            sa.Column('import_job_id', UUID(as_uuid=True), sa.ForeignKey('import_jobs.id'), nullable=False),
+            sa.Column('rolled_back_by', UUID(as_uuid=True), sa.ForeignKey('users.id'), nullable=False),
+            sa.Column('rolled_back_at', sa.DateTime(), nullable=False),
+            sa.Column('deleted_user_count', sa.Integer(), nullable=False),
+            sa.Column('deleted_user_ids', sa.JSON(), nullable=False),
+            sa.Column('reason', sa.Text(), nullable=True),
+            sa.Column('created_at', sa.DateTime(), nullable=False)
+        )
+
+
+def downgrade() -> None:
+    """Remove all added columns and tables"""
+
+    # Drop import_rollback_audit table
+    op.drop_table('import_rollback_audit')
+
+    # Drop import_jobs columns
+    op.drop_column('import_jobs', 'rollback_by')
+    op.drop_column('import_jobs', 'rollback_at')
+    op.drop_column('import_jobs', 'imported_user_ids')
+    op.drop_column('import_jobs', 'wordpress_metadata')
+    op.drop_column('import_jobs', 'field_mapping')
+
+    # Drop subscriptions columns
+    op.drop_column('subscriptions', 'manual_payment')
+    op.drop_column('subscriptions', 'donation_cents')
+    op.drop_column('subscriptions', 'base_subscription_cents')
+
+    # Drop events columns
+    op.drop_column('events', 'calendar_uid')
+
+    # Drop users columns (in reverse order)
+    op.drop_column('users', 'wordpress_registered_date')
+    op.drop_column('users', 'wordpress_user_id')
+    op.drop_column('users', 'import_job_id')
+    op.drop_column('users', 'import_source')
+    op.drop_column('users', 'rejected_by')
+    op.drop_column('users', 'rejected_at')
+    op.drop_column('users', 'rejection_reason')
+    op.drop_column('users', 'last_renewal_reminder_at')
+    op.drop_column('users', 'renewal_reminders_sent')
+    op.drop_column('users', 'last_payment_reminder_at')
+    op.drop_column('users', 'payment_reminders_sent')
+    op.drop_column('users', 'last_event_attendance_reminder_at')
+    op.drop_column('users', 'event_attendance_reminders_sent')
+    op.drop_column('users', 'last_email_verification_reminder_at')
+    op.drop_column('users', 'email_verification_reminders_sent')
+    op.drop_column('users', 'member_since')
+    op.drop_column('users', 'tos_accepted_at')
+    op.drop_column('users', 'accepts_tos')
+    op.drop_column('users', 'force_password_change')
+    op.drop_column('users', 'password_reset_expires')
+    op.drop_column('users', 'password_reset_token')
+    op.drop_column('users', 'show_in_directory')
+    op.drop_column('users', 'scholarship_requested')
+    op.drop_column('users', 'volunteer_interests')
+    op.drop_column('users', 'newsletter_publish_none')
+    op.drop_column('users', 'newsletter_publish_birthday')
+    op.drop_column('users', 'newsletter_publish_photo')
+    op.drop_column('users', 'newsletter_publish_name')

check_all_columns.sql

@@ -0,0 +1,92 @@
+-- Comprehensive check for all missing columns
+-- Run: psql -h 10.9.23.11 -p 54321 -U postgres -d loaf_new -f check_all_columns.sql
+
+\echo '================================================================'
+\echo 'COMPREHENSIVE COLUMN CHECK FOR ALL TABLES'
+\echo '================================================================'
+
+-- ============================================================
+-- 1. USERS TABLE
+-- ============================================================
+\echo ''
+\echo '1. USERS TABLE - Expected: 60+ columns'
+\echo 'Checking for specific columns:'
+
+SELECT
+    CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'newsletter_publish_name') THEN '✓' ELSE '✗' END || ' newsletter_publish_name',
+    CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'volunteer_interests') THEN '✓' ELSE '✗' END || ' volunteer_interests',
+    CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'scholarship_requested') THEN '✓' ELSE '✗' END || ' scholarship_requested',
+    CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'show_in_directory') THEN '✓' ELSE '✗' END || ' show_in_directory',
+    CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'password_reset_token') THEN '✓' ELSE '✗' END || ' password_reset_token',
+    CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'accepts_tos') THEN '✓' ELSE '✗' END || ' accepts_tos',
+    CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'member_since') THEN '✓' ELSE '✗' END || ' member_since',
+    CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'rejection_reason') THEN '✓' ELSE '✗' END || ' rejection_reason',
+    CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'import_source') THEN '✓' ELSE '✗' END || ' import_source'
+\gx
+
+-- ============================================================
+-- 2. EVENTS TABLE
+-- ============================================================
+\echo ''
+\echo '2. EVENTS TABLE'
+
+SELECT
+    CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'events' AND column_name = 'calendar_uid') THEN '✓' ELSE '✗' END || ' calendar_uid';
+
+-- ============================================================
+-- 3. SUBSCRIPTIONS TABLE
+-- ============================================================
+\echo ''
+\echo '3. SUBSCRIPTIONS TABLE'
+
+SELECT
+    CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'subscriptions' AND column_name = 'base_subscription_cents') THEN '✓' ELSE '✗' END || ' base_subscription_cents',
+    CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'subscriptions' AND column_name = 'donation_cents') THEN '✓' ELSE '✗' END || ' donation_cents',
+    CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'subscriptions' AND column_name = 'manual_payment') THEN '✓' ELSE '✗' END || ' manual_payment'
+\gx
+
+-- ============================================================
+-- 4. IMPORT_JOBS TABLE
+-- ============================================================
+\echo ''
+\echo '4. IMPORT_JOBS TABLE'
+
+SELECT
+    CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'import_jobs' AND column_name = 'field_mapping') THEN '✓' ELSE '✗' END || ' field_mapping',
+    CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'import_jobs' AND column_name = 'wordpress_metadata') THEN '✓' ELSE '✗' END || ' wordpress_metadata',
+    CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'import_jobs' AND column_name = 'imported_user_ids') THEN '✓' ELSE '✗' END || ' imported_user_ids',
+    CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'import_jobs' AND column_name = 'rollback_at') THEN '✓' ELSE '✗' END || ' rollback_at',
+    CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'import_jobs' AND column_name = 'rollback_by') THEN '✓' ELSE '✗' END || ' rollback_by'
+\gx
+
+-- ============================================================
+-- 5. CHECK IF IMPORT_ROLLBACK_AUDIT TABLE EXISTS
+-- ============================================================
+\echo ''
+\echo '5. IMPORT_ROLLBACK_AUDIT TABLE - Should exist'
+SELECT CASE
+    WHEN EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'import_rollback_audit')
+    THEN '✓ Table exists'
+    ELSE '✗ TABLE MISSING - Need to create it'
+END AS status;
+
+-- ============================================================
+-- SUMMARY: Count existing columns in each table
+-- ============================================================
+\echo ''
+\echo '================================================================'
+\echo 'SUMMARY: Column counts per table'
+\echo '================================================================'
+
+SELECT
+    table_name,
+    COUNT(*) as column_count
+FROM information_schema.columns
+WHERE table_name IN (
+    'users', 'events', 'event_rsvps', 'subscription_plans', 'subscriptions',
+    'donations', 'event_galleries', 'newsletter_archives', 'financial_reports',
+    'bylaws_documents', 'storage_usage', 'permissions', 'roles', 'role_permissions',
+    'user_invitations', 'import_jobs', 'import_rollback_audit'
+)
+GROUP BY table_name
+ORDER BY table_name;