diff --git a/alembic/versions/009_add_all_missing_columns.py b/alembic/versions/009_add_all_missing_columns.py new file mode 100644 index 0000000..a667f26 --- /dev/null +++ b/alembic/versions/009_add_all_missing_columns.py @@ -0,0 +1,237 @@ +"""add_all_missing_columns + +Revision ID: 009_add_all_missing +Revises: 008_add_donations +Create Date: 2026-01-04 + +Fixes: +- Add ALL remaining missing columns across all tables +- Users: newsletter preferences, volunteer, scholarship, directory, password reset, ToS, member_since, reminders, rejection, import tracking +- Events: calendar_uid +- Subscriptions: base_subscription_cents, donation_cents, manual_payment +- ImportJobs: WordPress import fields +- Create ImportRollbackAudit table if not exists +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy import inspect + + +# revision identifiers, used by Alembic. +revision: str = '009_add_all_missing' +down_revision: Union[str, None] = '008_add_donations' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Add all missing columns across all tables""" + + conn = op.get_bind() + inspector = inspect(conn) + + # ============================================================ + # 1. USERS TABLE - Add ~28 missing columns + # ============================================================ + users_columns = {col['name'] for col in inspector.get_columns('users')} + + # Newsletter publication preferences + if 'newsletter_publish_name' not in users_columns: + op.add_column('users', sa.Column('newsletter_publish_name', sa.Boolean(), nullable=False, server_default='false')) + if 'newsletter_publish_photo' not in users_columns: + op.add_column('users', sa.Column('newsletter_publish_photo', sa.Boolean(), nullable=False, server_default='false')) + if 'newsletter_publish_birthday' not in users_columns: + op.add_column('users', sa.Column('newsletter_publish_birthday', sa.Boolean(), nullable=False, server_default='false')) + if 'newsletter_publish_none' not in users_columns: + op.add_column('users', sa.Column('newsletter_publish_none', sa.Boolean(), nullable=False, server_default='false')) + + # Volunteer interests + if 'volunteer_interests' not in users_columns: + op.add_column('users', sa.Column('volunteer_interests', sa.JSON(), nullable=True, server_default='[]')) + + # Scholarship + if 'scholarship_requested' not in users_columns: + op.add_column('users', sa.Column('scholarship_requested', sa.Boolean(), nullable=False, server_default='false')) + + # Directory + if 'show_in_directory' not in users_columns: + op.add_column('users', sa.Column('show_in_directory', sa.Boolean(), nullable=False, server_default='false')) + + # Password reset + if 'password_reset_token' not in users_columns: + op.add_column('users', sa.Column('password_reset_token', sa.String(), nullable=True)) + if 'password_reset_expires' not in users_columns: + op.add_column('users', sa.Column('password_reset_expires', sa.DateTime(), nullable=True)) + if 'force_password_change' not in users_columns: + op.add_column('users', sa.Column('force_password_change', sa.Boolean(), nullable=False, server_default='false')) + + # Terms of Service + if 'accepts_tos' not in users_columns: + op.add_column('users', sa.Column('accepts_tos', sa.Boolean(), nullable=False, server_default='false')) + if 'tos_accepted_at' not in users_columns: + op.add_column('users', sa.Column('tos_accepted_at', sa.DateTime(), nullable=True)) + + # Member since + if 'member_since' not in users_columns: + op.add_column('users', sa.Column('member_since', sa.DateTime(), nullable=True)) + + # Email verification reminders + if 'email_verification_reminders_sent' not in users_columns: + op.add_column('users', sa.Column('email_verification_reminders_sent', sa.Integer(), nullable=False, server_default='0')) + if 'last_email_verification_reminder_at' not in users_columns: + op.add_column('users', sa.Column('last_email_verification_reminder_at', sa.DateTime(), nullable=True)) + + # Event attendance reminders + if 'event_attendance_reminders_sent' not in users_columns: + op.add_column('users', sa.Column('event_attendance_reminders_sent', sa.Integer(), nullable=False, server_default='0')) + if 'last_event_attendance_reminder_at' not in users_columns: + op.add_column('users', sa.Column('last_event_attendance_reminder_at', sa.DateTime(), nullable=True)) + + # Payment reminders + if 'payment_reminders_sent' not in users_columns: + op.add_column('users', sa.Column('payment_reminders_sent', sa.Integer(), nullable=False, server_default='0')) + if 'last_payment_reminder_at' not in users_columns: + op.add_column('users', sa.Column('last_payment_reminder_at', sa.DateTime(), nullable=True)) + + # Renewal reminders + if 'renewal_reminders_sent' not in users_columns: + op.add_column('users', sa.Column('renewal_reminders_sent', sa.Integer(), nullable=False, server_default='0')) + if 'last_renewal_reminder_at' not in users_columns: + op.add_column('users', sa.Column('last_renewal_reminder_at', sa.DateTime(), nullable=True)) + + # Rejection tracking + if 'rejection_reason' not in users_columns: + op.add_column('users', sa.Column('rejection_reason', sa.Text(), nullable=True)) + if 'rejected_at' not in users_columns: + op.add_column('users', sa.Column('rejected_at', sa.DateTime(timezone=True), nullable=True)) + if 'rejected_by' not in users_columns: + op.add_column('users', sa.Column('rejected_by', UUID(as_uuid=True), nullable=True)) + # Note: Foreign key constraint skipped to avoid circular dependency issues + + # WordPress import tracking + if 'import_source' not in users_columns: + op.add_column('users', sa.Column('import_source', sa.String(50), nullable=True)) + if 'import_job_id' not in users_columns: + op.add_column('users', sa.Column('import_job_id', UUID(as_uuid=True), nullable=True)) + # Note: Foreign key will be added after import_jobs table is updated + if 'wordpress_user_id' not in users_columns: + op.add_column('users', sa.Column('wordpress_user_id', sa.BigInteger(), nullable=True)) + if 'wordpress_registered_date' not in users_columns: + op.add_column('users', sa.Column('wordpress_registered_date', sa.DateTime(timezone=True), nullable=True)) + + # ============================================================ + # 2. EVENTS TABLE - Add calendar_uid + # ============================================================ + events_columns = {col['name'] for col in inspector.get_columns('events')} + + if 'calendar_uid' not in events_columns: + op.add_column('events', sa.Column('calendar_uid', sa.String(), nullable=True)) + + # ============================================================ + # 3. SUBSCRIPTIONS TABLE - Add donation tracking + # ============================================================ + subscriptions_columns = {col['name'] for col in inspector.get_columns('subscriptions')} + + if 'base_subscription_cents' not in subscriptions_columns: + op.add_column('subscriptions', sa.Column('base_subscription_cents', sa.Integer(), nullable=True)) + # Update existing rows: base_subscription_cents = amount_paid_cents - donation_cents (default 0) + op.execute("UPDATE subscriptions SET base_subscription_cents = COALESCE(amount_paid_cents, 0) WHERE base_subscription_cents IS NULL") + # Make it non-nullable after populating + op.alter_column('subscriptions', 'base_subscription_cents', nullable=False) + + if 'donation_cents' not in subscriptions_columns: + op.add_column('subscriptions', sa.Column('donation_cents', sa.Integer(), nullable=False, server_default='0')) + + if 'manual_payment' not in subscriptions_columns: + op.add_column('subscriptions', sa.Column('manual_payment', sa.Boolean(), nullable=False, server_default='false')) + + # ============================================================ + # 4. IMPORT_JOBS TABLE - Add WordPress import fields + # ============================================================ + import_jobs_columns = {col['name'] for col in inspector.get_columns('import_jobs')} + + if 'field_mapping' not in import_jobs_columns: + op.add_column('import_jobs', sa.Column('field_mapping', sa.JSON(), nullable=False, server_default='{}')) + + if 'wordpress_metadata' not in import_jobs_columns: + op.add_column('import_jobs', sa.Column('wordpress_metadata', sa.JSON(), nullable=False, server_default='{}')) + + if 'imported_user_ids' not in import_jobs_columns: + op.add_column('import_jobs', sa.Column('imported_user_ids', sa.JSON(), nullable=False, server_default='[]')) + + if 'rollback_at' not in import_jobs_columns: + op.add_column('import_jobs', sa.Column('rollback_at', sa.DateTime(), nullable=True)) + + if 'rollback_by' not in import_jobs_columns: + op.add_column('import_jobs', sa.Column('rollback_by', UUID(as_uuid=True), nullable=True)) + # Foreign key will be added if needed + + # ============================================================ + # 5. CREATE IMPORT_ROLLBACK_AUDIT TABLE + # ============================================================ + if 'import_rollback_audit' not in inspector.get_table_names(): + op.create_table( + 'import_rollback_audit', + sa.Column('id', UUID(as_uuid=True), primary_key=True), + sa.Column('import_job_id', UUID(as_uuid=True), sa.ForeignKey('import_jobs.id'), nullable=False), + sa.Column('rolled_back_by', UUID(as_uuid=True), sa.ForeignKey('users.id'), nullable=False), + sa.Column('rolled_back_at', sa.DateTime(), nullable=False), + sa.Column('deleted_user_count', sa.Integer(), nullable=False), + sa.Column('deleted_user_ids', sa.JSON(), nullable=False), + sa.Column('reason', sa.Text(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False) + ) + + +def downgrade() -> None: + """Remove all added columns and tables""" + + # Drop import_rollback_audit table + op.drop_table('import_rollback_audit') + + # Drop import_jobs columns + op.drop_column('import_jobs', 'rollback_by') + op.drop_column('import_jobs', 'rollback_at') + op.drop_column('import_jobs', 'imported_user_ids') + op.drop_column('import_jobs', 'wordpress_metadata') + op.drop_column('import_jobs', 'field_mapping') + + # Drop subscriptions columns + op.drop_column('subscriptions', 'manual_payment') + op.drop_column('subscriptions', 'donation_cents') + op.drop_column('subscriptions', 'base_subscription_cents') + + # Drop events columns + op.drop_column('events', 'calendar_uid') + + # Drop users columns (in reverse order) + op.drop_column('users', 'wordpress_registered_date') + op.drop_column('users', 'wordpress_user_id') + op.drop_column('users', 'import_job_id') + op.drop_column('users', 'import_source') + op.drop_column('users', 'rejected_by') + op.drop_column('users', 'rejected_at') + op.drop_column('users', 'rejection_reason') + op.drop_column('users', 'last_renewal_reminder_at') + op.drop_column('users', 'renewal_reminders_sent') + op.drop_column('users', 'last_payment_reminder_at') + op.drop_column('users', 'payment_reminders_sent') + op.drop_column('users', 'last_event_attendance_reminder_at') + op.drop_column('users', 'event_attendance_reminders_sent') + op.drop_column('users', 'last_email_verification_reminder_at') + op.drop_column('users', 'email_verification_reminders_sent') + op.drop_column('users', 'member_since') + op.drop_column('users', 'tos_accepted_at') + op.drop_column('users', 'accepts_tos') + op.drop_column('users', 'force_password_change') + op.drop_column('users', 'password_reset_expires') + op.drop_column('users', 'password_reset_token') + op.drop_column('users', 'show_in_directory') + op.drop_column('users', 'scholarship_requested') + op.drop_column('users', 'volunteer_interests') + op.drop_column('users', 'newsletter_publish_none') + op.drop_column('users', 'newsletter_publish_birthday') + op.drop_column('users', 'newsletter_publish_photo') + op.drop_column('users', 'newsletter_publish_name') diff --git a/check_all_columns.sql b/check_all_columns.sql new file mode 100644 index 0000000..a4c0dc3 --- /dev/null +++ b/check_all_columns.sql @@ -0,0 +1,92 @@ +-- Comprehensive check for all missing columns +-- Run: psql -h 10.9.23.11 -p 54321 -U postgres -d loaf_new -f check_all_columns.sql + +\echo '================================================================' +\echo 'COMPREHENSIVE COLUMN CHECK FOR ALL TABLES' +\echo '================================================================' + +-- ============================================================ +-- 1. USERS TABLE +-- ============================================================ +\echo '' +\echo '1. USERS TABLE - Expected: 60+ columns' +\echo 'Checking for specific columns:' + +SELECT + CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'newsletter_publish_name') THEN '✓' ELSE '✗' END || ' newsletter_publish_name', + CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'volunteer_interests') THEN '✓' ELSE '✗' END || ' volunteer_interests', + CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'scholarship_requested') THEN '✓' ELSE '✗' END || ' scholarship_requested', + CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'show_in_directory') THEN '✓' ELSE '✗' END || ' show_in_directory', + CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'password_reset_token') THEN '✓' ELSE '✗' END || ' password_reset_token', + CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'accepts_tos') THEN '✓' ELSE '✗' END || ' accepts_tos', + CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'member_since') THEN '✓' ELSE '✗' END || ' member_since', + CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'rejection_reason') THEN '✓' ELSE '✗' END || ' rejection_reason', + CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'import_source') THEN '✓' ELSE '✗' END || ' import_source' +\gx + +-- ============================================================ +-- 2. EVENTS TABLE +-- ============================================================ +\echo '' +\echo '2. EVENTS TABLE' + +SELECT + CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'events' AND column_name = 'calendar_uid') THEN '✓' ELSE '✗' END || ' calendar_uid'; + +-- ============================================================ +-- 3. SUBSCRIPTIONS TABLE +-- ============================================================ +\echo '' +\echo '3. SUBSCRIPTIONS TABLE' + +SELECT + CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'subscriptions' AND column_name = 'base_subscription_cents') THEN '✓' ELSE '✗' END || ' base_subscription_cents', + CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'subscriptions' AND column_name = 'donation_cents') THEN '✓' ELSE '✗' END || ' donation_cents', + CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'subscriptions' AND column_name = 'manual_payment') THEN '✓' ELSE '✗' END || ' manual_payment' +\gx + +-- ============================================================ +-- 4. IMPORT_JOBS TABLE +-- ============================================================ +\echo '' +\echo '4. IMPORT_JOBS TABLE' + +SELECT + CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'import_jobs' AND column_name = 'field_mapping') THEN '✓' ELSE '✗' END || ' field_mapping', + CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'import_jobs' AND column_name = 'wordpress_metadata') THEN '✓' ELSE '✗' END || ' wordpress_metadata', + CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'import_jobs' AND column_name = 'imported_user_ids') THEN '✓' ELSE '✗' END || ' imported_user_ids', + CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'import_jobs' AND column_name = 'rollback_at') THEN '✓' ELSE '✗' END || ' rollback_at', + CASE WHEN EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name = 'import_jobs' AND column_name = 'rollback_by') THEN '✓' ELSE '✗' END || ' rollback_by' +\gx + +-- ============================================================ +-- 5. CHECK IF IMPORT_ROLLBACK_AUDIT TABLE EXISTS +-- ============================================================ +\echo '' +\echo '5. IMPORT_ROLLBACK_AUDIT TABLE - Should exist' +SELECT CASE + WHEN EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'import_rollback_audit') + THEN '✓ Table exists' + ELSE '✗ TABLE MISSING - Need to create it' +END AS status; + +-- ============================================================ +-- SUMMARY: Count existing columns in each table +-- ============================================================ +\echo '' +\echo '================================================================' +\echo 'SUMMARY: Column counts per table' +\echo '================================================================' + +SELECT + table_name, + COUNT(*) as column_count +FROM information_schema.columns +WHERE table_name IN ( + 'users', 'events', 'event_rsvps', 'subscription_plans', 'subscriptions', + 'donations', 'event_galleries', 'newsletter_archives', 'financial_reports', + 'bylaws_documents', 'storage_usage', 'permissions', 'roles', 'role_permissions', + 'user_invitations', 'import_jobs', 'import_rollback_audit' +) +GROUP BY table_name +ORDER BY table_name;