diff --git a/__pycache__/r2_storage.cpython-312.pyc b/__pycache__/r2_storage.cpython-312.pyc index 863f476..cd7ebd1 100644 Binary files a/__pycache__/r2_storage.cpython-312.pyc and b/__pycache__/r2_storage.cpython-312.pyc differ diff --git a/__pycache__/server.cpython-312.pyc b/__pycache__/server.cpython-312.pyc index 0c295b8..70dafa8 100644 Binary files a/__pycache__/server.cpython-312.pyc and b/__pycache__/server.cpython-312.pyc differ diff --git a/__pycache__/wordpress_parser.cpython-312.pyc b/__pycache__/wordpress_parser.cpython-312.pyc index 8028514..a37394a 100644 Binary files a/__pycache__/wordpress_parser.cpython-312.pyc and b/__pycache__/wordpress_parser.cpython-312.pyc differ diff --git a/import_templates.py b/import_templates.py new file mode 100644 index 0000000..15bd1f3 --- /dev/null +++ b/import_templates.py @@ -0,0 +1,1133 @@ +""" +CSV Import Templates Module + +This module provides standardized CSV templates for importing data into the +membership platform. Templates are designed to be source-agnostic, allowing +clients to transform their data from any system to match our schema. + +Supported Templates: +- users.csv - Core user data +- subscriptions.csv - Membership/plan history +- donations.csv - Donation history +- payments.csv - Payment transactions +- registration_data.csv - Custom registration fields + +Author: Claude Code +Date: 2026-02-03 +""" + +import csv +import io +from typing import Dict, List, Any, Optional, Tuple +from datetime import datetime, date +import re +import logging + +logger = logging.getLogger(__name__) + +# ============================================================================= +# Template Definitions +# ============================================================================= + +TEMPLATES = { + "users": { + "name": "Users", + "description": "Core user data including personal information, contact details, and account settings", + "required": True, + "filename": "users_template.csv", + "columns": [ + # Required fields + {"name": "email", "required": True, "type": "email", "description": "User's email address (unique identifier)"}, + {"name": "first_name", "required": True, "type": "string", "description": "First name"}, + {"name": "last_name", "required": True, "type": "string", "description": "Last name"}, + + # Account settings + {"name": "status", "required": False, "type": "enum", "description": "Account status", + "options": ["active", "inactive", "pending_email", "pending_validation", "pre_validated", "payment_pending", "canceled", "expired"], + "default": "active"}, + {"name": "role", "required": False, "type": "enum", "description": "User role", + "options": ["member", "admin", "finance", "guest"], + "default": "member"}, + + # Contact info + {"name": "phone", "required": False, "type": "string", "description": "Phone number"}, + {"name": "address", "required": False, "type": "string", "description": "Street address"}, + {"name": "city", "required": False, "type": "string", "description": "City"}, + {"name": "state", "required": False, "type": "string", "description": "State/Province"}, + {"name": "zipcode", "required": False, "type": "string", "description": "ZIP/Postal code"}, + + # Personal info + {"name": "date_of_birth", "required": False, "type": "date", "description": "Date of birth (YYYY-MM-DD)"}, + {"name": "member_since", "required": False, "type": "date", "description": "Member since date (YYYY-MM-DD)"}, + + # Partner info + {"name": "partner_first_name", "required": False, "type": "string", "description": "Partner's first name"}, + {"name": "partner_last_name", "required": False, "type": "string", "description": "Partner's last name"}, + {"name": "partner_is_member", "required": False, "type": "boolean", "description": "Is partner a member? (true/false)"}, + + # Referral + {"name": "referred_by_member_name", "required": False, "type": "string", "description": "Name of referring member"}, + {"name": "lead_sources", "required": False, "type": "list", "description": "How they heard about us (comma-separated)"}, + + # Directory settings + {"name": "show_in_directory", "required": False, "type": "boolean", "description": "Show in member directory? (true/false)", "default": "false"}, + {"name": "directory_display_name", "required": False, "type": "string", "description": "Display name for directory"}, + {"name": "directory_bio", "required": False, "type": "string", "description": "Bio for directory"}, + {"name": "directory_email", "required": False, "type": "boolean", "description": "Show email in directory? (true/false)"}, + {"name": "directory_phone", "required": False, "type": "boolean", "description": "Show phone in directory? (true/false)"}, + {"name": "directory_address", "required": False, "type": "boolean", "description": "Show address in directory? (true/false)"}, + + # Newsletter preferences + {"name": "newsletter_subscribed", "required": False, "type": "boolean", "description": "Subscribed to newsletter? (true/false)"}, + {"name": "newsletter_publish_name", "required": False, "type": "boolean", "description": "Publish name in newsletter? (true/false)"}, + {"name": "newsletter_publish_photo", "required": False, "type": "boolean", "description": "Publish photo in newsletter? (true/false)"}, + {"name": "newsletter_publish_birthday", "required": False, "type": "boolean", "description": "Publish birthday in newsletter? (true/false)"}, + + # Volunteer interests + {"name": "volunteer_interests", "required": False, "type": "list", "description": "Volunteer interests (comma-separated)"}, + + # Scholarship + {"name": "scholarship_requested", "required": False, "type": "boolean", "description": "Requested scholarship? (true/false)"}, + {"name": "scholarship_reason", "required": False, "type": "string", "description": "Reason for scholarship request"}, + + # Social media + {"name": "social_media_facebook", "required": False, "type": "string", "description": "Facebook profile URL"}, + {"name": "social_media_instagram", "required": False, "type": "string", "description": "Instagram handle"}, + {"name": "social_media_twitter", "required": False, "type": "string", "description": "Twitter/X handle"}, + {"name": "social_media_linkedin", "required": False, "type": "string", "description": "LinkedIn profile URL"}, + ], + "example_rows": [ + { + "email": "john.doe@example.com", + "first_name": "John", + "last_name": "Doe", + "status": "active", + "role": "member", + "phone": "555-123-4567", + "address": "123 Main St", + "city": "Houston", + "state": "TX", + "zipcode": "77001", + "date_of_birth": "1985-06-15", + "member_since": "2023-01-01", + "partner_first_name": "Jane", + "partner_last_name": "Doe", + "partner_is_member": "true", + "referred_by_member_name": "", + "lead_sources": "Friend,Social Media", + "show_in_directory": "true", + "directory_display_name": "John D.", + "directory_bio": "Active community member", + "directory_email": "true", + "directory_phone": "false", + "directory_address": "false", + "newsletter_subscribed": "true", + "newsletter_publish_name": "true", + "newsletter_publish_photo": "false", + "newsletter_publish_birthday": "true", + "volunteer_interests": "Events,Outreach", + "scholarship_requested": "false", + "scholarship_reason": "", + "social_media_facebook": "", + "social_media_instagram": "@johndoe", + "social_media_twitter": "", + "social_media_linkedin": "", + }, + { + "email": "jane.smith@example.com", + "first_name": "Jane", + "last_name": "Smith", + "status": "active", + "role": "member", + "phone": "555-987-6543", + "address": "456 Oak Ave", + "city": "Dallas", + "state": "TX", + "zipcode": "75201", + "date_of_birth": "1990-03-22", + "member_since": "2024-06-15", + "partner_first_name": "", + "partner_last_name": "", + "partner_is_member": "false", + "referred_by_member_name": "John Doe", + "lead_sources": "Current member", + "show_in_directory": "false", + "directory_display_name": "", + "directory_bio": "", + "directory_email": "false", + "directory_phone": "false", + "directory_address": "false", + "newsletter_subscribed": "true", + "newsletter_publish_name": "false", + "newsletter_publish_photo": "false", + "newsletter_publish_birthday": "false", + "volunteer_interests": "", + "scholarship_requested": "false", + "scholarship_reason": "", + "social_media_facebook": "", + "social_media_instagram": "", + "social_media_twitter": "", + "social_media_linkedin": "", + }, + ] + }, + + "subscriptions": { + "name": "Subscriptions", + "description": "Membership and subscription history for users", + "required": False, + "filename": "subscriptions_template.csv", + "columns": [ + {"name": "email", "required": True, "type": "email", "description": "User's email (must match users.csv)"}, + {"name": "plan_name", "required": True, "type": "string", "description": "Name of the subscription plan"}, + {"name": "status", "required": False, "type": "enum", "description": "Subscription status", + "options": ["active", "cancelled", "expired", "past_due"], + "default": "active"}, + {"name": "amount", "required": False, "type": "decimal", "description": "Subscription amount paid"}, + {"name": "currency", "required": False, "type": "string", "description": "Currency code (e.g., USD)", "default": "USD"}, + {"name": "start_date", "required": False, "type": "date", "description": "Subscription start date (YYYY-MM-DD)"}, + {"name": "end_date", "required": False, "type": "date", "description": "Subscription end date (YYYY-MM-DD)"}, + {"name": "auto_renew", "required": False, "type": "boolean", "description": "Auto-renew enabled? (true/false)", "default": "false"}, + {"name": "payment_method", "required": False, "type": "string", "description": "Payment method used"}, + {"name": "notes", "required": False, "type": "string", "description": "Additional notes"}, + # Stripe transaction fields + {"name": "stripe_subscription_id", "required": False, "type": "string", "description": "Stripe Subscription ID (sub_xxx)"}, + {"name": "stripe_customer_id", "required": False, "type": "string", "description": "Stripe Customer ID (cus_xxx)"}, + {"name": "stripe_payment_intent_id", "required": False, "type": "string", "description": "Stripe Payment Intent ID (pi_xxx)"}, + {"name": "stripe_invoice_id", "required": False, "type": "string", "description": "Stripe Invoice ID (in_xxx)"}, + {"name": "stripe_charge_id", "required": False, "type": "string", "description": "Stripe Charge ID (ch_xxx)"}, + {"name": "stripe_receipt_url", "required": False, "type": "string", "description": "Stripe receipt URL"}, + {"name": "card_last4", "required": False, "type": "string", "description": "Last 4 digits of card used"}, + {"name": "card_brand", "required": False, "type": "string", "description": "Card brand (Visa, Mastercard, etc.)"}, + ], + "example_rows": [ + { + "email": "john.doe@example.com", + "plan_name": "Annual Membership", + "status": "active", + "amount": "100.00", + "currency": "USD", + "start_date": "2024-01-01", + "end_date": "2024-12-31", + "auto_renew": "true", + "payment_method": "Credit Card", + "notes": "", + "stripe_subscription_id": "sub_1234567890", + "stripe_customer_id": "cus_1234567890", + "stripe_payment_intent_id": "pi_1234567890", + "stripe_invoice_id": "in_1234567890", + "stripe_charge_id": "ch_1234567890", + "stripe_receipt_url": "https://pay.stripe.com/receipts/...", + "card_last4": "4242", + "card_brand": "Visa", + }, + { + "email": "john.doe@example.com", + "plan_name": "Annual Membership", + "status": "expired", + "amount": "85.00", + "currency": "USD", + "start_date": "2023-01-01", + "end_date": "2023-12-31", + "auto_renew": "false", + "payment_method": "Check", + "notes": "First year member discount", + "stripe_subscription_id": "", + "stripe_customer_id": "", + "stripe_payment_intent_id": "", + "stripe_invoice_id": "", + "stripe_charge_id": "", + "stripe_receipt_url": "", + "card_last4": "", + "card_brand": "", + }, + { + "email": "jane.smith@example.com", + "plan_name": "Monthly Membership", + "status": "active", + "amount": "15.00", + "currency": "USD", + "start_date": "2024-06-15", + "end_date": "2024-07-15", + "auto_renew": "true", + "payment_method": "Credit Card", + "notes": "", + "stripe_subscription_id": "sub_0987654321", + "stripe_customer_id": "cus_0987654321", + "stripe_payment_intent_id": "pi_0987654321", + "stripe_invoice_id": "", + "stripe_charge_id": "ch_0987654321", + "stripe_receipt_url": "", + "card_last4": "1234", + "card_brand": "Mastercard", + }, + ] + }, + + "donations": { + "name": "Donations", + "description": "Donation history for users and anonymous donors", + "required": False, + "filename": "donations_template.csv", + "columns": [ + {"name": "email", "required": False, "type": "email", "description": "Donor's email (leave blank for anonymous)"}, + {"name": "donor_name", "required": False, "type": "string", "description": "Donor's name (for anonymous donations)"}, + {"name": "amount", "required": True, "type": "decimal", "description": "Donation amount"}, + {"name": "currency", "required": False, "type": "string", "description": "Currency code", "default": "USD"}, + {"name": "date", "required": True, "type": "date", "description": "Donation date (YYYY-MM-DD)"}, + {"name": "type", "required": False, "type": "enum", "description": "Donation type", + "options": ["member", "public"], + "default": "member"}, + {"name": "status", "required": False, "type": "enum", "description": "Donation status", + "options": ["completed", "pending", "failed", "refunded"], + "default": "completed"}, + {"name": "payment_method", "required": False, "type": "string", "description": "Payment method"}, + {"name": "campaign", "required": False, "type": "string", "description": "Campaign or fund name"}, + {"name": "notes", "required": False, "type": "string", "description": "Additional notes"}, + {"name": "is_recurring", "required": False, "type": "boolean", "description": "Is this a recurring donation?", "default": "false"}, + # Stripe transaction fields + {"name": "stripe_payment_intent_id", "required": False, "type": "string", "description": "Stripe Payment Intent ID (pi_xxx)"}, + {"name": "stripe_charge_id", "required": False, "type": "string", "description": "Stripe Charge ID (ch_xxx)"}, + {"name": "stripe_customer_id", "required": False, "type": "string", "description": "Stripe Customer ID (cus_xxx)"}, + {"name": "stripe_checkout_session_id", "required": False, "type": "string", "description": "Stripe Checkout Session ID (cs_xxx)"}, + {"name": "stripe_receipt_url", "required": False, "type": "string", "description": "Stripe receipt URL"}, + {"name": "card_last4", "required": False, "type": "string", "description": "Last 4 digits of card used"}, + {"name": "card_brand", "required": False, "type": "string", "description": "Card brand (Visa, Mastercard, etc.)"}, + ], + "example_rows": [ + { + "email": "john.doe@example.com", + "donor_name": "", + "amount": "50.00", + "currency": "USD", + "date": "2024-03-15", + "type": "member", + "status": "completed", + "payment_method": "Credit Card", + "campaign": "Annual Fund", + "notes": "", + "is_recurring": "false", + "stripe_payment_intent_id": "pi_donation123", + "stripe_charge_id": "ch_donation123", + "stripe_customer_id": "cus_1234567890", + "stripe_checkout_session_id": "", + "stripe_receipt_url": "https://pay.stripe.com/receipts/...", + "card_last4": "4242", + "card_brand": "Visa", + }, + { + "email": "", + "donor_name": "Anonymous Supporter", + "amount": "100.00", + "currency": "USD", + "date": "2024-04-01", + "type": "public", + "status": "completed", + "payment_method": "Check", + "campaign": "Building Fund", + "notes": "Mailed check", + "is_recurring": "false", + "stripe_payment_intent_id": "", + "stripe_charge_id": "", + "stripe_customer_id": "", + "stripe_checkout_session_id": "", + "stripe_receipt_url": "", + "card_last4": "", + "card_brand": "", + }, + { + "email": "jane.smith@example.com", + "donor_name": "", + "amount": "25.00", + "currency": "USD", + "date": "2024-05-01", + "type": "member", + "status": "completed", + "payment_method": "Credit Card", + "campaign": "", + "notes": "Monthly recurring", + "is_recurring": "true", + "stripe_payment_intent_id": "pi_recurring456", + "stripe_charge_id": "ch_recurring456", + "stripe_customer_id": "cus_0987654321", + "stripe_checkout_session_id": "cs_checkout789", + "stripe_receipt_url": "", + "card_last4": "1234", + "card_brand": "Mastercard", + }, + ] + }, + + "payments": { + "name": "Payments", + "description": "Payment transaction history", + "required": False, + "filename": "payments_template.csv", + "columns": [ + {"name": "email", "required": True, "type": "email", "description": "User's email (must match users.csv)"}, + {"name": "amount", "required": True, "type": "decimal", "description": "Payment amount"}, + {"name": "currency", "required": False, "type": "string", "description": "Currency code", "default": "USD"}, + {"name": "date", "required": True, "type": "date", "description": "Payment date (YYYY-MM-DD)"}, + {"name": "type", "required": False, "type": "enum", "description": "Payment type", + "options": ["subscription", "donation", "event", "other"], + "default": "subscription"}, + {"name": "status", "required": False, "type": "enum", "description": "Payment status", + "options": ["completed", "pending", "failed", "refunded"], + "default": "completed"}, + {"name": "payment_method", "required": False, "type": "string", "description": "Payment method (Credit Card, Check, Cash, etc.)"}, + {"name": "description", "required": False, "type": "string", "description": "Payment description"}, + {"name": "notes", "required": False, "type": "string", "description": "Internal notes"}, + # Stripe transaction fields + {"name": "stripe_payment_intent_id", "required": False, "type": "string", "description": "Stripe Payment Intent ID (pi_xxx)"}, + {"name": "stripe_charge_id", "required": False, "type": "string", "description": "Stripe Charge ID (ch_xxx)"}, + {"name": "stripe_customer_id", "required": False, "type": "string", "description": "Stripe Customer ID (cus_xxx)"}, + {"name": "stripe_invoice_id", "required": False, "type": "string", "description": "Stripe Invoice ID (in_xxx)"}, + {"name": "stripe_receipt_url", "required": False, "type": "string", "description": "Stripe receipt URL"}, + {"name": "card_last4", "required": False, "type": "string", "description": "Last 4 digits of card used"}, + {"name": "card_brand", "required": False, "type": "string", "description": "Card brand (Visa, Mastercard, etc.)"}, + ], + "example_rows": [ + { + "email": "john.doe@example.com", + "amount": "100.00", + "currency": "USD", + "date": "2024-01-01", + "type": "subscription", + "status": "completed", + "payment_method": "Credit Card", + "description": "Annual Membership 2024", + "notes": "", + "stripe_payment_intent_id": "pi_sub123", + "stripe_charge_id": "ch_sub123", + "stripe_customer_id": "cus_1234567890", + "stripe_invoice_id": "in_sub123", + "stripe_receipt_url": "https://pay.stripe.com/receipts/...", + "card_last4": "4242", + "card_brand": "Visa", + }, + { + "email": "john.doe@example.com", + "amount": "50.00", + "currency": "USD", + "date": "2024-03-15", + "type": "donation", + "status": "completed", + "payment_method": "Credit Card", + "description": "Annual Fund Donation", + "notes": "", + "stripe_payment_intent_id": "pi_don456", + "stripe_charge_id": "ch_don456", + "stripe_customer_id": "cus_1234567890", + "stripe_invoice_id": "", + "stripe_receipt_url": "", + "card_last4": "4242", + "card_brand": "Visa", + }, + { + "email": "jane.smith@example.com", + "amount": "15.00", + "currency": "USD", + "date": "2024-06-15", + "type": "subscription", + "status": "completed", + "payment_method": "Credit Card", + "description": "Monthly Membership - June 2024", + "notes": "", + "stripe_payment_intent_id": "pi_mon789", + "stripe_charge_id": "ch_mon789", + "stripe_customer_id": "cus_0987654321", + "stripe_invoice_id": "in_mon789", + "stripe_receipt_url": "https://pay.stripe.com/receipts/...", + "card_last4": "1234", + "card_brand": "Mastercard", + }, + ] + }, + + # Note: registration_data template is generated dynamically from the registration schema + # See generate_dynamic_registration_template() function + "registration_data": { + "name": "Custom Registration Data", + "description": "Custom registration fields based on current registration form configuration. This template is generated dynamically.", + "required": False, + "filename": "registration_data_template.csv", + "is_dynamic": True, # Flag to indicate this template is generated dynamically + "columns": [ + {"name": "email", "required": True, "type": "email", "description": "User's email (must match users.csv)"}, + # Additional columns are generated dynamically from registration schema + ], + "example_rows": [ + { + "email": "john.doe@example.com", + # Example values are generated dynamically + }, + ] + }, +} + + +# ============================================================================= +# Template Generation Functions +# ============================================================================= + +def generate_template_csv(template_type: str, include_examples: bool = True) -> str: + """ + Generate a CSV template string for the given template type. + + Args: + template_type: One of 'users', 'subscriptions', 'donations', 'payments', 'registration_data' + include_examples: Whether to include example data rows + + Returns: + CSV string with headers and optionally example data + """ + if template_type not in TEMPLATES: + raise ValueError(f"Unknown template type: {template_type}") + + template = TEMPLATES[template_type] + columns = template["columns"] + + output = io.StringIO() + writer = csv.writer(output) + + # Write header row + headers = [col["name"] for col in columns] + writer.writerow(headers) + + # Write example rows if requested + if include_examples and template.get("example_rows"): + for example in template["example_rows"]: + row = [example.get(col["name"], "") for col in columns] + writer.writerow(row) + + return output.getvalue() + + +def generate_template_documentation(template_type: str) -> Dict[str, Any]: + """ + Generate documentation for a template including field descriptions. + + Args: + template_type: One of the template types + + Returns: + Dictionary with template metadata and field documentation + """ + if template_type not in TEMPLATES: + raise ValueError(f"Unknown template type: {template_type}") + + template = TEMPLATES[template_type] + + return { + "type": template_type, + "name": template["name"], + "description": template["description"], + "required": template["required"], + "filename": template["filename"], + "fields": [ + { + "name": col["name"], + "required": col["required"], + "type": col["type"], + "description": col["description"], + "options": col.get("options"), + "default": col.get("default"), + } + for col in template["columns"] + ] + } + + +def get_all_templates_info() -> List[Dict[str, Any]]: + """Get summary information about all available templates.""" + return [ + { + "type": key, + "name": template["name"], + "description": template["description"], + "required": template["required"], + "filename": template["filename"], + "field_count": len(template["columns"]), + "required_fields": [col["name"] for col in template["columns"] if col["required"]], + } + for key, template in TEMPLATES.items() + ] + + +# ============================================================================= +# CSV Parsing and Validation Functions +# ============================================================================= + +def parse_boolean(value: Any) -> Optional[bool]: + """Parse a boolean value from CSV.""" + if value is None or value == "": + return None + if isinstance(value, bool): + return value + str_val = str(value).lower().strip() + if str_val in ("true", "yes", "1", "y"): + return True + if str_val in ("false", "no", "0", "n"): + return False + return None + + +def parse_date(value: Any) -> Optional[date]: + """Parse a date value from CSV (expects YYYY-MM-DD).""" + if value is None or value == "": + return None + try: + if isinstance(value, date): + return value + return datetime.strptime(str(value).strip(), "%Y-%m-%d").date() + except ValueError: + # Try alternative formats + for fmt in ["%m/%d/%Y", "%d/%m/%Y", "%Y/%m/%d"]: + try: + return datetime.strptime(str(value).strip(), fmt).date() + except ValueError: + continue + return None + + +def parse_decimal(value: Any) -> Optional[float]: + """Parse a decimal value from CSV.""" + if value is None or value == "": + return None + try: + # Remove currency symbols and commas + cleaned = re.sub(r'[,$€£]', '', str(value).strip()) + return float(cleaned) + except ValueError: + return None + + +def parse_list(value: Any) -> List[str]: + """Parse a comma-separated list from CSV.""" + if value is None or value == "": + return [] + return [item.strip() for item in str(value).split(",") if item.strip()] + + +def validate_email(email: str) -> bool: + """Validate email format.""" + pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$' + return bool(re.match(pattern, email.strip())) + + +def validate_row(row: Dict[str, Any], template_type: str) -> Tuple[bool, List[str], Dict[str, Any]]: + """ + Validate a single row against the template schema. + + Args: + row: Dictionary of column name -> value + template_type: The template type to validate against + + Returns: + Tuple of (is_valid, errors, parsed_row) + """ + if template_type not in TEMPLATES: + return False, [f"Unknown template type: {template_type}"], {} + + template = TEMPLATES[template_type] + columns = {col["name"]: col for col in template["columns"]} + errors = [] + parsed_row = {} + + # Check required fields + for col_name, col_def in columns.items(): + value = row.get(col_name, "") + + # Check required + if col_def["required"] and (value is None or str(value).strip() == ""): + errors.append(f"Required field '{col_name}' is missing") + continue + + # Skip empty optional fields + if value is None or str(value).strip() == "": + parsed_row[col_name] = col_def.get("default") + continue + + # Type validation and parsing + col_type = col_def["type"] + + if col_type == "email": + if not validate_email(str(value)): + errors.append(f"Invalid email format in '{col_name}': {value}") + else: + parsed_row[col_name] = str(value).strip().lower() + + elif col_type == "string": + parsed_row[col_name] = str(value).strip() + + elif col_type == "boolean": + parsed = parse_boolean(value) + if parsed is None and str(value).strip(): + errors.append(f"Invalid boolean in '{col_name}': {value} (use true/false)") + else: + parsed_row[col_name] = parsed + + elif col_type == "date": + parsed = parse_date(value) + if parsed is None: + errors.append(f"Invalid date in '{col_name}': {value} (use YYYY-MM-DD)") + else: + parsed_row[col_name] = parsed + + elif col_type == "decimal": + parsed = parse_decimal(value) + if parsed is None: + errors.append(f"Invalid number in '{col_name}': {value}") + else: + parsed_row[col_name] = parsed + + elif col_type == "list": + parsed_row[col_name] = parse_list(value) + + elif col_type == "enum": + str_val = str(value).strip().lower() + options = [opt.lower() for opt in col_def.get("options", [])] + if str_val and str_val not in options: + errors.append(f"Invalid value in '{col_name}': {value}. Must be one of: {', '.join(col_def['options'])}") + else: + parsed_row[col_name] = str_val if str_val else col_def.get("default") + else: + parsed_row[col_name] = value + + return len(errors) == 0, errors, parsed_row + + +def parse_csv_file(file_content: bytes, template_type: str) -> Dict[str, Any]: + """ + Parse and validate a CSV file against a template. + + Args: + file_content: Raw CSV file content + template_type: The template type to validate against + + Returns: + Dictionary with parsing results: + { + 'success': bool, + 'total_rows': int, + 'valid_rows': int, + 'invalid_rows': int, + 'rows': [...], # Parsed and validated rows + 'errors': [...], # List of {row: int, errors: [...]} + 'warnings': [...], + } + """ + result = { + 'success': True, + 'total_rows': 0, + 'valid_rows': 0, + 'invalid_rows': 0, + 'rows': [], + 'errors': [], + 'warnings': [], + } + + try: + # Decode file content + try: + content = file_content.decode('utf-8') + except UnicodeDecodeError: + content = file_content.decode('latin-1') + + # Parse CSV + reader = csv.DictReader(io.StringIO(content)) + + # Check if headers match template + template = TEMPLATES.get(template_type) + if not template: + result['success'] = False + result['errors'].append({'row': 0, 'errors': [f"Unknown template type: {template_type}"]}) + return result + + expected_headers = {col["name"] for col in template["columns"]} + actual_headers = set(reader.fieldnames or []) + + # Check for missing required headers + required_headers = {col["name"] for col in template["columns"] if col["required"]} + missing_required = required_headers - actual_headers + if missing_required: + result['success'] = False + result['errors'].append({ + 'row': 0, + 'errors': [f"Missing required columns: {', '.join(missing_required)}"] + }) + return result + + # Warn about unexpected headers + unexpected = actual_headers - expected_headers + if unexpected: + result['warnings'].append(f"Unexpected columns will be ignored: {', '.join(unexpected)}") + + # Parse and validate each row + for row_num, row in enumerate(reader, start=2): # Start at 2 (1 is header) + result['total_rows'] += 1 + + is_valid, errors, parsed_row = validate_row(row, template_type) + parsed_row['_row_number'] = row_num + + if is_valid: + result['valid_rows'] += 1 + result['rows'].append(parsed_row) + else: + result['invalid_rows'] += 1 + result['errors'].append({ + 'row': row_num, + 'errors': errors, + 'data': {k: v for k, v in row.items() if k in expected_headers} + }) + + if result['invalid_rows'] > 0: + result['success'] = False + + except Exception as e: + logger.error(f"Error parsing CSV: {str(e)}") + result['success'] = False + result['errors'].append({'row': 0, 'errors': [f"Error parsing CSV: {str(e)}"]}) + + return result + + +def cross_validate_files( + users_result: Dict[str, Any], + subscriptions_result: Optional[Dict[str, Any]] = None, + donations_result: Optional[Dict[str, Any]] = None, + payments_result: Optional[Dict[str, Any]] = None, + registration_data_result: Optional[Dict[str, Any]] = None, + existing_emails: Optional[set] = None +) -> Dict[str, Any]: + """ + Cross-validate multiple CSV files to ensure referential integrity. + + Checks: + - All emails in related files exist in users.csv + - No duplicate emails in users.csv + - Emails don't already exist in database (if existing_emails provided) + + Returns: + Validation result with warnings and errors + """ + result = { + 'valid': True, + 'warnings': [], + 'errors': [], + 'summary': { + 'total_users': 0, + 'new_users': 0, + 'duplicate_users': 0, + 'total_subscriptions': 0, + 'total_donations': 0, + 'total_payments': 0, + 'total_registration_fields': 0, + 'orphaned_records': 0, + } + } + + # Get all user emails from users.csv + user_emails = set() + duplicate_emails = set() + + for row in users_result.get('rows', []): + email = row.get('email', '').lower() + if email: + if email in user_emails: + duplicate_emails.add(email) + user_emails.add(email) + + result['summary']['total_users'] = len(user_emails) + result['summary']['duplicate_users'] = len(duplicate_emails) + + if duplicate_emails: + result['errors'].append(f"Duplicate emails in users.csv: {', '.join(list(duplicate_emails)[:5])}{'...' if len(duplicate_emails) > 5 else ''}") + result['valid'] = False + + # Check against existing database emails + if existing_emails: + existing_in_import = user_emails & existing_emails + new_emails = user_emails - existing_emails + result['summary']['new_users'] = len(new_emails) + + if existing_in_import: + result['warnings'].append( + f"{len(existing_in_import)} email(s) already exist in database and will be updated: " + f"{', '.join(list(existing_in_import)[:3])}{'...' if len(existing_in_import) > 3 else ''}" + ) + + # Validate subscriptions + if subscriptions_result and subscriptions_result.get('rows'): + result['summary']['total_subscriptions'] = len(subscriptions_result['rows']) + orphaned = [] + for row in subscriptions_result['rows']: + email = row.get('email', '').lower() + if email and email not in user_emails: + orphaned.append(email) + if orphaned: + result['warnings'].append( + f"{len(orphaned)} subscription(s) reference emails not in users.csv: " + f"{', '.join(orphaned[:3])}{'...' if len(orphaned) > 3 else ''}" + ) + result['summary']['orphaned_records'] += len(orphaned) + + # Validate donations (email is optional for anonymous) + if donations_result and donations_result.get('rows'): + result['summary']['total_donations'] = len(donations_result['rows']) + orphaned = [] + for row in donations_result['rows']: + email = row.get('email', '').lower() + if email and email not in user_emails: + orphaned.append(email) + if orphaned: + result['warnings'].append( + f"{len(orphaned)} donation(s) reference emails not in users.csv: " + f"{', '.join(orphaned[:3])}{'...' if len(orphaned) > 3 else ''}" + ) + result['summary']['orphaned_records'] += len(orphaned) + + # Validate payments + if payments_result and payments_result.get('rows'): + result['summary']['total_payments'] = len(payments_result['rows']) + orphaned = [] + for row in payments_result['rows']: + email = row.get('email', '').lower() + if email and email not in user_emails: + orphaned.append(email) + if orphaned: + result['warnings'].append( + f"{len(orphaned)} payment(s) reference emails not in users.csv: " + f"{', '.join(orphaned[:3])}{'...' if len(orphaned) > 3 else ''}" + ) + result['summary']['orphaned_records'] += len(orphaned) + + # Validate registration data + if registration_data_result and registration_data_result.get('rows'): + result['summary']['total_registration_fields'] = len(registration_data_result['rows']) + orphaned = [] + for row in registration_data_result['rows']: + email = row.get('email', '').lower() + if email and email not in user_emails: + orphaned.append(email) + if orphaned: + result['warnings'].append( + f"{len(orphaned)} registration field(s) reference emails not in users.csv: " + f"{', '.join(orphaned[:3])}{'...' if len(orphaned) > 3 else ''}" + ) + result['summary']['orphaned_records'] += len(orphaned) + + return result + + +# ============================================================================= +# Dynamic Registration Template Generator +# ============================================================================= + +# Standard User model fields that have direct mappings (not stored in custom_registration_data) +USER_MODEL_FIELDS = { + 'email', 'first_name', 'last_name', 'phone', 'address', 'city', 'state', 'zipcode', + 'date_of_birth', 'lead_sources', 'partner_first_name', 'partner_last_name', + 'partner_is_member', 'partner_plan_to_become_member', 'referred_by_member_name', + 'newsletter_subscribed', 'newsletter_publish_name', 'newsletter_publish_photo', + 'newsletter_publish_birthday', 'newsletter_publish_none', 'volunteer_interests', + 'scholarship_requested', 'scholarship_reason', 'show_in_directory', + 'directory_email', 'directory_bio', 'directory_address', 'directory_phone', + 'directory_dob', 'directory_partner_name', 'social_media_facebook', + 'social_media_instagram', 'social_media_twitter', 'social_media_linkedin', + 'accepts_tos', 'password' +} + + +def extract_custom_fields_from_schema(registration_schema: dict) -> List[Dict[str, Any]]: + """ + Extract custom fields from registration schema that should be stored in custom_registration_data. + + Custom fields are those where: + - The field has no 'mapping' to a User model column, OR + - The field's 'mapping' is not in USER_MODEL_FIELDS + + Args: + registration_schema: The registration form schema from SystemSettings + + Returns: + List of custom field definitions with their metadata + """ + custom_fields = [] + + for step in registration_schema.get('steps', []): + for section in step.get('sections', []): + for field in section.get('fields', []): + field_id = field.get('id', '') + mapping = field.get('mapping', field_id) + + # Skip fixed/core fields that map directly to User model + if mapping in USER_MODEL_FIELDS: + continue + + # Skip email/password/tos which are handled specially + if field_id in ('email', 'password', 'confirm_password', 'accepts_tos'): + continue + + # This is a custom field + field_type = field.get('type', 'text') + + # Map registration field types to CSV column types + csv_type = 'string' + if field_type in ('number', 'currency'): + csv_type = 'decimal' + elif field_type == 'date': + csv_type = 'date' + elif field_type in ('checkbox', 'toggle'): + csv_type = 'boolean' + elif field_type in ('select', 'radio'): + csv_type = 'enum' + elif field_type in ('multiselect', 'checkbox_group'): + csv_type = 'list' + + custom_field = { + 'name': field_id, + 'required': field.get('required', False), + 'type': csv_type, + 'label': field.get('label', field_id), + 'description': field.get('placeholder', field.get('label', f'Custom field: {field_id}')), + 'section': section.get('title', 'Unknown'), + 'step': step.get('title', 'Unknown'), + } + + # Add options for enum types + if field_type in ('select', 'radio', 'multiselect', 'checkbox_group'): + options = field.get('options', []) + if options: + custom_field['options'] = [opt.get('value', opt.get('label', '')) for opt in options] + + custom_fields.append(custom_field) + + return custom_fields + + +def generate_dynamic_registration_template(registration_schema: dict) -> Dict[str, Any]: + """ + Generate a dynamic registration_data template based on the current registration schema. + + This creates a template where: + - email column is always first and required + - Each custom field becomes its own column (flat structure, not key-value pairs) + - Column names match the field IDs from the registration schema + + Args: + registration_schema: The registration form schema + + Returns: + Template definition compatible with TEMPLATES format + """ + custom_fields = extract_custom_fields_from_schema(registration_schema) + + # Build columns list - email is always first + columns = [ + {"name": "email", "required": True, "type": "email", "description": "User's email (must match users.csv)"} + ] + + # Add custom field columns + for field in custom_fields: + col = { + "name": field['name'], + "required": field['required'], + "type": field['type'], + "description": f"{field['label']} ({field['section']})", + } + if field.get('options'): + col['options'] = field['options'] + columns.append(col) + + # Generate example rows + example_rows = [] + if custom_fields: + example_row = {"email": "john.doe@example.com"} + for field in custom_fields: + # Generate example value based on type + if field['type'] == 'boolean': + example_row[field['name']] = 'true' + elif field['type'] == 'date': + example_row[field['name']] = '2024-01-15' + elif field['type'] == 'decimal': + example_row[field['name']] = '100.00' + elif field['type'] == 'list': + opts = field.get('options', ['Option1', 'Option2']) + example_row[field['name']] = ','.join(opts[:2]) if len(opts) > 1 else opts[0] if opts else '' + elif field['type'] == 'enum': + opts = field.get('options', []) + example_row[field['name']] = opts[0] if opts else 'value' + else: + example_row[field['name']] = f'Example {field["label"]}' + example_rows.append(example_row) + + # Add second example row + example_row2 = {"email": "jane.smith@example.com"} + for field in custom_fields: + if field['type'] == 'boolean': + example_row2[field['name']] = 'false' + elif field['type'] == 'date': + example_row2[field['name']] = '2023-06-20' + elif field['type'] == 'decimal': + example_row2[field['name']] = '50.00' + elif field['type'] == 'list': + opts = field.get('options', ['Option1']) + example_row2[field['name']] = opts[0] if opts else '' + elif field['type'] == 'enum': + opts = field.get('options', []) + example_row2[field['name']] = opts[1] if len(opts) > 1 else opts[0] if opts else '' + else: + example_row2[field['name']] = '' + example_rows.append(example_row2) + + return { + "name": "Custom Registration Data", + "description": f"Custom registration fields from current form configuration ({len(custom_fields)} custom fields)", + "required": False, + "filename": "registration_data_template.csv", + "is_dynamic": True, + "columns": columns, + "example_rows": example_rows, + "custom_fields_info": [ + { + "name": f['name'], + "label": f['label'], + "type": f['type'], + "required": f['required'], + "section": f['section'], + "step": f['step'], + } + for f in custom_fields + ] + } + + +def generate_template_csv_dynamic(registration_schema: dict, include_examples: bool = True) -> str: + """ + Generate CSV content for the dynamic registration template. + + Args: + registration_schema: The registration form schema + include_examples: Whether to include example data rows + + Returns: + CSV string content + """ + template = generate_dynamic_registration_template(registration_schema) + columns = template["columns"] + + output = io.StringIO() + writer = csv.writer(output) + + # Write header row + headers = [col["name"] for col in columns] + writer.writerow(headers) + + # Write example rows if requested + if include_examples and template.get("example_rows"): + for example in template["example_rows"]: + row = [example.get(col["name"], "") for col in columns] + writer.writerow(row) + + return output.getvalue() diff --git a/r2_storage.py b/r2_storage.py index 699d859..639ed03 100644 --- a/r2_storage.py +++ b/r2_storage.py @@ -50,6 +50,14 @@ class R2Storage: 'image/svg+xml': ['.svg'] } + # CSV files for imports + ALLOWED_CSV_TYPES = { + 'text/csv': ['.csv'], + 'text/plain': ['.csv'], # Some systems report CSV as text/plain + 'application/csv': ['.csv'], + 'application/vnd.ms-excel': ['.csv'], # Old Excel type sometimes used for CSV + } + def __init__(self): """Initialize R2 client with credentials from environment""" self.account_id = os.getenv('R2_ACCOUNT_ID') @@ -240,6 +248,127 @@ class R2Storage: except ClientError: return False + async def upload_bytes( + self, + content: bytes, + folder: str, + filename: str, + content_type: str = 'text/csv' + ) -> tuple[str, str, int]: + """ + Upload raw bytes to R2 storage (useful for CSV imports) + + Args: + content: Raw bytes to upload + folder: Folder path in R2 (e.g., 'imports/job-id') + filename: Original filename + content_type: MIME type of the content + + Returns: + tuple: (public_url, object_key, file_size_bytes) + + Raises: + HTTPException: If upload fails + """ + try: + file_size = len(content) + + # Generate unique filename preserving original extension + file_extension = Path(filename).suffix.lower() or '.csv' + unique_filename = f"{uuid.uuid4()}{file_extension}" + object_key = f"{folder}/{unique_filename}" + + # Upload to R2 + self.client.put_object( + Bucket=self.bucket_name, + Key=object_key, + Body=content, + ContentType=content_type, + ContentLength=file_size + ) + + # Generate public URL + public_url = self.get_public_url(object_key) + + return public_url, object_key, file_size + + except ClientError as e: + raise HTTPException( + status_code=500, + detail=f"Failed to upload to R2: {str(e)}" + ) + except Exception as e: + raise HTTPException( + status_code=500, + detail=f"Upload error: {str(e)}" + ) + + async def download_file(self, object_key: str) -> bytes: + """ + Download a file from R2 storage + + Args: + object_key: The S3 object key (path) of the file + + Returns: + bytes: File content + + Raises: + HTTPException: If download fails + """ + try: + response = self.client.get_object( + Bucket=self.bucket_name, + Key=object_key + ) + return response['Body'].read() + + except ClientError as e: + if e.response['Error']['Code'] == 'NoSuchKey': + raise HTTPException(status_code=404, detail="File not found in storage") + raise HTTPException( + status_code=500, + detail=f"Failed to download file from R2: {str(e)}" + ) + except Exception as e: + raise HTTPException( + status_code=500, + detail=f"Download error: {str(e)}" + ) + + async def delete_multiple(self, object_keys: list[str]) -> bool: + """ + Delete multiple files from R2 storage + + Args: + object_keys: List of S3 object keys to delete + + Returns: + bool: True if successful + + Raises: + HTTPException: If deletion fails + """ + if not object_keys: + return True + + try: + # R2/S3 delete_objects accepts up to 1000 keys at once + objects = [{'Key': key} for key in object_keys if key] + + if objects: + self.client.delete_objects( + Bucket=self.bucket_name, + Delete={'Objects': objects} + ) + return True + + except ClientError as e: + raise HTTPException( + status_code=500, + detail=f"Failed to delete files from R2: {str(e)}" + ) + # Singleton instance _r2_storage = None diff --git a/server.py b/server.py index 2aaf852..5b73862 100644 --- a/server.py +++ b/server.py @@ -5,7 +5,7 @@ from sqlalchemy.orm import Session from sqlalchemy import or_ from pydantic import BaseModel, EmailStr, Field, validator from typing import List, Optional, Literal -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta, timezone, date from dotenv import load_dotenv from pathlib import Path from contextlib import asynccontextmanager @@ -16,6 +16,7 @@ import secrets import csv import io import json +import tempfile from database import engine, get_db, Base from models import User, Event, EventRSVP, UserStatus, UserRole, RSVPStatus, SubscriptionPlan, Subscription, SubscriptionStatus, StorageUsage, EventGallery, NewsletterArchive, FinancialReport, BylawsDocument, Permission, RolePermission, Role, UserInvitation, InvitationStatus, ImportJob, ImportJobStatus, ImportRollbackAudit, Donation, DonationType, DonationStatus, SystemSettings, PaymentMethod, PaymentMethodType @@ -131,6 +132,30 @@ def set_user_role(user: User, role_enum: UserRole, db: Session): else: logger.warning(f"Role not found for code: {role_enum.value}") + +def serialize_for_json(obj): + """ + Recursively convert datetime objects to ISO strings for JSON serialization. + Handles nested dicts, lists, and other common types. + """ + if obj is None: + return None + elif isinstance(obj, datetime): + return obj.isoformat() + elif isinstance(obj, date): + return obj.isoformat() + elif isinstance(obj, dict): + return {k: serialize_for_json(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [serialize_for_json(item) for item in obj] + elif isinstance(obj, (str, int, float, bool)): + return obj + elif hasattr(obj, '__dict__'): + return serialize_for_json(obj.__dict__) + else: + return str(obj) + + # ============================================================ # Pydantic Models # ============================================================ @@ -3198,6 +3223,86 @@ async def export_users_csv( } ) + +# Import Jobs endpoints (must be before /admin/users/{user_id} to avoid route conflict) +@api_router.get("/admin/users/import-jobs") +async def get_import_jobs( + status: Optional[str] = None, + current_user: User = Depends(require_permission("users.view")), + db: Session = Depends(get_db) +): + """ + List all import jobs with optional status filter + Admin/Superadmin only + Requires permission: users.view + """ + query = db.query(ImportJob) + + if status: + try: + status_enum = ImportJobStatus[status] + query = query.filter(ImportJob.status == status_enum) + except KeyError: + raise HTTPException(status_code=400, detail=f"Invalid status: {status}") + + jobs = query.order_by(ImportJob.started_at.desc()).all() + + return [ + { + "id": str(job.id), + "filename": job.filename, + "total_rows": job.total_rows, + "processed_rows": job.processed_rows, + "successful_rows": job.successful_rows, + "failed_rows": job.failed_rows, + "status": job.status.value, + "imported_by": str(job.imported_by), + "started_at": job.started_at.isoformat(), + "completed_at": job.completed_at.isoformat() if job.completed_at else None, + "error_count": len(job.errors) if job.errors else 0 + } + for job in jobs + ] + + +@api_router.get("/admin/users/import-jobs/{job_id}") +async def get_import_job_details( + job_id: str, + current_user: User = Depends(require_permission("users.view")), + db: Session = Depends(get_db) +): + """ + Get detailed information about a specific import job + Admin/Superadmin only + Requires permission: users.view + """ + job = db.query(ImportJob).filter(ImportJob.id == job_id).first() + if not job: + raise HTTPException(status_code=404, detail="Import job not found") + + # Get importer details + importer = db.query(User).filter(User.id == job.imported_by).first() + + return { + "id": str(job.id), + "filename": job.filename, + "total_rows": job.total_rows, + "processed_rows": job.processed_rows, + "successful_rows": job.successful_rows, + "failed_rows": job.failed_rows, + "status": job.status.value, + "imported_by": { + "id": str(importer.id), + "email": importer.email, + "name": f"{importer.first_name} {importer.last_name}" + } if importer else None, + "started_at": job.started_at.isoformat(), + "completed_at": job.completed_at.isoformat() if job.completed_at else None, + "errors": job.errors or [], # Full error list + "imported_user_ids": [str(uid) for uid in (job.imported_user_ids or [])] # User IDs for bulk actions + } + + @api_router.get("/admin/users/{user_id}") async def get_user_by_id( user_id: str, @@ -3624,6 +3729,122 @@ async def admin_reset_user_password( return {"message": f"Password reset for {user.email}. Temporary password emailed."} + +class BulkPasswordResetRequest(BaseModel): + user_ids: Optional[List[str]] = None # Specific user IDs to reset + import_job_id: Optional[str] = None # Reset all users from this import job + filter_status: Optional[str] = None # Filter by user status + send_email: bool = True # Whether to send reset emails + + +@api_router.post("/admin/users/bulk-password-reset") +async def bulk_password_reset( + request: BulkPasswordResetRequest, + current_user: User = Depends(require_permission("users.reset_password")), + db: Session = Depends(get_db) +): + """ + Send password reset emails to multiple users at once. + + Options: + - user_ids: List of specific user IDs to reset + - import_job_id: Reset all users from a specific import job + - filter_status: Only reset users with this status + - send_email: Whether to send reset emails (default True) + + All selected users will have force_password_change set to True. + + Requires permission: users.reset_password + """ + # Build query based on filters + query = db.query(User) + + if request.user_ids: + # Specific user IDs + try: + user_uuids = [uuid.UUID(uid) for uid in request.user_ids] + query = query.filter(User.id.in_(user_uuids)) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid user ID format") + + elif request.import_job_id: + # All users from import job + try: + job_uuid = uuid.UUID(request.import_job_id) + query = query.filter(User.import_job_id == job_uuid) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid import job ID format") + else: + raise HTTPException( + status_code=400, + detail="Must provide either user_ids or import_job_id" + ) + + # Apply status filter if provided + if request.filter_status: + try: + status_enum = UserStatus[request.filter_status] + query = query.filter(User.status == status_enum) + except KeyError: + raise HTTPException(status_code=400, detail=f"Invalid status: {request.filter_status}") + + # Get users + users = query.all() + + if not users: + raise HTTPException(status_code=404, detail="No users found matching criteria") + + # Limit to prevent timeout (max 200 users at once) + if len(users) > 200: + raise HTTPException( + status_code=400, + detail=f"Too many users ({len(users)}). Maximum 200 users per request. Use filters to narrow down." + ) + + # Process each user + successful = 0 + failed = 0 + errors = [] + + for user in users: + try: + # Set force_password_change flag + user.force_password_change = True + + if request.send_email: + # Generate reset token and send email + reset_token = create_password_reset_token(user.email) + reset_url = f"{os.getenv('FRONTEND_URL')}/reset-password?token={reset_token}" + await send_password_reset_email(user.email, user.first_name, reset_url) + + successful += 1 + + except Exception as e: + failed += 1 + errors.append({ + 'user_id': str(user.id), + 'email': user.email, + 'error': str(e) + }) + logger.error(f"Failed to send reset email to {user.email}: {str(e)}") + + # Commit all force_password_change updates + db.commit() + + logger.info( + f"Bulk password reset by {current_user.email}: " + f"{successful} successful, {failed} failed" + ) + + return { + 'total_users': len(users), + 'successful': successful, + 'failed': failed, + 'emails_sent': successful if request.send_email else 0, + 'errors': errors[:20] # Limit error response + } + + @api_router.put("/admin/users/{user_id}/role") async def change_user_role( user_id: str, @@ -4401,85 +4622,811 @@ async def import_users_csv( } -@api_router.get("/admin/users/import-jobs") -async def get_import_jobs( - status: Optional[str] = None, - current_user: User = Depends(require_permission("users.view")), +# ============================================================================ +# Template-Based CSV Import Endpoints +# ============================================================================ + +@api_router.get("/admin/import/templates") +async def get_import_templates( + current_user: User = Depends(require_permission("users.import")) +): + """ + Get list of all available import templates with their documentation. + + Returns list of templates with: + - type, name, description + - required flag (users.csv is required, others optional) + - field definitions with types and validation rules + + Requires permission: users.import + """ + from import_templates import get_all_templates_info + return get_all_templates_info() + + +@api_router.get("/admin/import/templates/{template_type}") +async def get_template_details( + template_type: str, + current_user: User = Depends(require_permission("users.import")) +): + """ + Get detailed documentation for a specific template type. + + Args: + template_type: One of 'users', 'subscriptions', 'donations', 'payments', 'registration_data' + + Returns: + Template metadata with all field definitions + + Requires permission: users.import + """ + from import_templates import generate_template_documentation, TEMPLATES + + if template_type not in TEMPLATES: + raise HTTPException( + status_code=400, + detail=f"Unknown template type: {template_type}. Valid types: {', '.join(TEMPLATES.keys())}" + ) + + return generate_template_documentation(template_type) + + +@api_router.get("/admin/import/templates/{template_type}/download") +async def download_template_csv( + template_type: str, + include_examples: bool = True, + current_user: User = Depends(require_permission("users.import")), db: Session = Depends(get_db) ): """ - List all import jobs with optional status filter - Admin/Superadmin only - Requires permission: users.view + Download a CSV template file. + + Args: + template_type: Template type to download + include_examples: Whether to include example data rows (default: True) + + Returns: + CSV file download + + Note: For registration_data, the template is generated dynamically based on + the current registration form configuration. + + Requires permission: users.import """ - query = db.query(ImportJob) + from import_templates import generate_template_csv, generate_template_csv_dynamic, TEMPLATES - if status: - try: - status_enum = ImportJobStatus[status] - query = query.filter(ImportJob.status == status_enum) - except KeyError: - raise HTTPException(status_code=400, detail=f"Invalid status: {status}") + if template_type not in TEMPLATES: + raise HTTPException( + status_code=400, + detail=f"Unknown template type: {template_type}" + ) - jobs = query.order_by(ImportJob.started_at.desc()).all() + # For registration_data, generate dynamic template based on current registration schema + if template_type == "registration_data": + registration_schema = get_registration_schema(db) + csv_content = generate_template_csv_dynamic(registration_schema, include_examples) + filename = "registration_data_template.csv" + else: + csv_content = generate_template_csv(template_type, include_examples) + filename = TEMPLATES[template_type]["filename"] - return [ - { - "id": str(job.id), - "filename": job.filename, - "total_rows": job.total_rows, - "processed_rows": job.processed_rows, - "successful_rows": job.successful_rows, - "failed_rows": job.failed_rows, - "status": job.status.value, - "imported_by": str(job.imported_by), - "started_at": job.started_at.isoformat(), - "completed_at": job.completed_at.isoformat() if job.completed_at else None, - "error_count": len(job.errors) if job.errors else 0 + return StreamingResponse( + iter([csv_content]), + media_type="text/csv", + headers={ + "Content-Disposition": f"attachment; filename={filename}" } - for job in jobs - ] + ) -@api_router.get("/admin/users/import-jobs/{job_id}") -async def get_import_job_details( - job_id: str, - current_user: User = Depends(require_permission("users.view")), +@api_router.get("/admin/import/templates/registration_data/info") +async def get_registration_template_info( + current_user: User = Depends(require_permission("users.import")), db: Session = Depends(get_db) ): """ - Get detailed information about a specific import job - Admin/Superadmin only - Requires permission: users.view + Get information about the dynamic registration_data template. + + Returns the template structure with all custom fields from the current + registration form configuration. + + Requires permission: users.import + """ + from import_templates import generate_dynamic_registration_template + + registration_schema = get_registration_schema(db) + template = generate_dynamic_registration_template(registration_schema) + + return { + "name": template["name"], + "description": template["description"], + "filename": template["filename"], + "is_dynamic": True, + "custom_fields_count": len(template.get("custom_fields_info", [])), + "columns": template["columns"], + "custom_fields": template.get("custom_fields_info", []), + "registration_schema_version": registration_schema.get("version", "unknown"), + } + + +@api_router.post("/admin/import/template/upload") +async def upload_template_import( + users_file: UploadFile = File(..., description="Users CSV file (required)"), + subscriptions_file: Optional[UploadFile] = File(None, description="Subscriptions CSV file (optional)"), + donations_file: Optional[UploadFile] = File(None, description="Donations CSV file (optional)"), + payments_file: Optional[UploadFile] = File(None, description="Payments CSV file (optional)"), + registration_data_file: Optional[UploadFile] = File(None, description="Registration data CSV file (optional)"), + current_user: User = Depends(require_permission("users.import")), + db: Session = Depends(get_db) +): + """ + Upload CSV files using standardized templates for import. + + This endpoint: + 1. Validates all uploaded CSV files against their template schemas + 2. Cross-validates referential integrity (emails match across files) + 3. Checks for duplicates and existing users + 4. Creates ImportJob record for tracking + 5. Returns preview summary with validation results + + Required: users_file + Optional: subscriptions_file, donations_file, payments_file, registration_data_file + + Returns: + Import job ID and validation summary + + Requires permission: users.import + """ + from import_templates import parse_csv_file, cross_validate_files + + # Validate users file is provided and is CSV + if not users_file.filename.endswith('.csv'): + raise HTTPException(status_code=400, detail="Users file must be a CSV") + + # Get existing emails for duplicate detection + existing_emails = set( + email[0].lower() for email in db.query(User.email).all() + ) + + # Parse all uploaded files + parse_results = {} + + # Parse users file (required) + users_content = await users_file.read() + parse_results['users'] = parse_csv_file(users_content, 'users') + + if not parse_results['users']['success'] and parse_results['users']['valid_rows'] == 0: + raise HTTPException( + status_code=400, + detail={ + "message": "Users file validation failed", + "errors": parse_results['users']['errors'][:10] + } + ) + + # Parse optional files + files_uploaded = {'users': users_file.filename} + + if subscriptions_file and subscriptions_file.filename: + if not subscriptions_file.filename.endswith('.csv'): + raise HTTPException(status_code=400, detail="Subscriptions file must be a CSV") + content = await subscriptions_file.read() + parse_results['subscriptions'] = parse_csv_file(content, 'subscriptions') + files_uploaded['subscriptions'] = subscriptions_file.filename + + if donations_file and donations_file.filename: + if not donations_file.filename.endswith('.csv'): + raise HTTPException(status_code=400, detail="Donations file must be a CSV") + content = await donations_file.read() + parse_results['donations'] = parse_csv_file(content, 'donations') + files_uploaded['donations'] = donations_file.filename + + if payments_file and payments_file.filename: + if not payments_file.filename.endswith('.csv'): + raise HTTPException(status_code=400, detail="Payments file must be a CSV") + content = await payments_file.read() + parse_results['payments'] = parse_csv_file(content, 'payments') + files_uploaded['payments'] = payments_file.filename + + if registration_data_file and registration_data_file.filename: + if not registration_data_file.filename.endswith('.csv'): + raise HTTPException(status_code=400, detail="Registration data file must be a CSV") + content = await registration_data_file.read() + parse_results['registration_data'] = parse_csv_file(content, 'registration_data') + files_uploaded['registration_data'] = registration_data_file.filename + + # Cross-validate all files + cross_validation = cross_validate_files( + users_result=parse_results['users'], + subscriptions_result=parse_results.get('subscriptions'), + donations_result=parse_results.get('donations'), + payments_result=parse_results.get('payments'), + registration_data_result=parse_results.get('registration_data'), + existing_emails=existing_emails + ) + + # Create import job + import_job = ImportJob( + filename=users_file.filename, + total_rows=parse_results['users']['total_rows'], + processed_rows=0, + successful_rows=0, + failed_rows=0, + status=ImportJobStatus.preview_ready, + imported_by=current_user.id, + wordpress_metadata={ + 'import_type': 'template', + 'files_uploaded': files_uploaded, + 'parse_results': { + key: { + 'total_rows': result['total_rows'], + 'valid_rows': result['valid_rows'], + 'invalid_rows': result['invalid_rows'], + 'errors': result['errors'][:20], # Limit stored errors + 'warnings': result.get('warnings', []), + } + for key, result in parse_results.items() + }, + 'cross_validation': cross_validation, + 'preview_data': { + 'users': parse_results['users']['rows'][:100], # Preview first 100 + 'subscriptions': parse_results.get('subscriptions', {}).get('rows', [])[:100], + 'donations': parse_results.get('donations', {}).get('rows', [])[:100], + 'payments': parse_results.get('payments', {}).get('rows', [])[:100], + 'registration_data': parse_results.get('registration_data', {}).get('rows', [])[:100], + }, + 'full_data': { + 'users': parse_results['users']['rows'], + 'subscriptions': parse_results.get('subscriptions', {}).get('rows', []), + 'donations': parse_results.get('donations', {}).get('rows', []), + 'payments': parse_results.get('payments', {}).get('rows', []), + 'registration_data': parse_results.get('registration_data', {}).get('rows', []), + } + }, + errors=[] + ) + + db.add(import_job) + db.commit() + db.refresh(import_job) + + logger.info(f"Template import job {import_job.id} created by {current_user.email}") + + return { + "import_job_id": str(import_job.id), + "files_uploaded": files_uploaded, + "validation": { + key: { + 'total_rows': result['total_rows'], + 'valid_rows': result['valid_rows'], + 'invalid_rows': result['invalid_rows'], + 'has_errors': result['invalid_rows'] > 0, + 'errors_preview': result['errors'][:5], + 'warnings': result.get('warnings', []), + } + for key, result in parse_results.items() + }, + "cross_validation": cross_validation, + "can_proceed": cross_validation['valid'] or parse_results['users']['valid_rows'] > 0, + } + + +@api_router.get("/admin/import/template/{job_id}/preview") +async def get_template_import_preview( + job_id: str, + file_type: str = "users", + page: int = 1, + page_size: int = 20, + current_user: User = Depends(require_permission("users.import")), + db: Session = Depends(get_db) +): + """ + Get paginated preview of parsed data from a template import job. + + Args: + job_id: Import job ID + file_type: Which file to preview ('users', 'subscriptions', 'donations', 'payments', 'registration_data') + page: Page number (1-indexed) + page_size: Items per page + + Returns: + Paginated preview data with row details + + Requires permission: users.import """ job = db.query(ImportJob).filter(ImportJob.id == job_id).first() if not job: raise HTTPException(status_code=404, detail="Import job not found") - # Get importer details - importer = db.query(User).filter(User.id == job.imported_by).first() + if job.status != ImportJobStatus.preview_ready: + raise HTTPException( + status_code=400, + detail=f"Import job is not in preview_ready status (current: {job.status.value})" + ) + + # Check if this is a template import + if job.wordpress_metadata.get('import_type') != 'template': + raise HTTPException( + status_code=400, + detail="This is not a template-based import job" + ) + + # Get preview data for requested file type + preview_data = job.wordpress_metadata.get('preview_data', {}) + full_data = job.wordpress_metadata.get('full_data', {}) + + # Use full data if available, otherwise preview + data = full_data.get(file_type, preview_data.get(file_type, [])) + + # Paginate + total_rows = len(data) + total_pages = (total_rows + page_size - 1) // page_size if total_rows > 0 else 1 + start_idx = (page - 1) * page_size + end_idx = start_idx + page_size + page_data = data[start_idx:end_idx] + + # Get parse results for this file type + parse_results = job.wordpress_metadata.get('parse_results', {}).get(file_type, {}) return { - "id": str(job.id), - "filename": job.filename, - "total_rows": job.total_rows, - "processed_rows": job.processed_rows, - "successful_rows": job.successful_rows, - "failed_rows": job.failed_rows, - "status": job.status.value, - "imported_by": { - "id": str(importer.id), - "email": importer.email, - "name": f"{importer.first_name} {importer.last_name}" - } if importer else None, - "started_at": job.started_at.isoformat(), - "completed_at": job.completed_at.isoformat() if job.completed_at else None, - "errors": job.errors or [] # Full error list + "job_id": str(job.id), + "file_type": file_type, + "page": page, + "page_size": page_size, + "total_rows": total_rows, + "total_pages": total_pages, + "valid_rows": parse_results.get('valid_rows', 0), + "invalid_rows": parse_results.get('invalid_rows', 0), + "rows": page_data, + "errors": parse_results.get('errors', []), + "warnings": parse_results.get('warnings', []), + } + + +@api_router.post("/admin/import/template/{job_id}/execute") +async def execute_template_import( + job_id: str, + options: dict = {}, + current_user: User = Depends(require_permission("users.import")), + db: Session = Depends(get_db) +): + """ + Execute template-based import. + + Options: + - skip_notifications: bool (default True) - Skip all email notifications + - update_existing: bool (default False) - Update existing users by email + - import_subscriptions: bool (default True) - Import subscription records + - import_donations: bool (default True) - Import donation records + - import_payments: bool (default True) - Import payment records + - import_registration_data: bool (default True) - Import custom registration fields + - skip_errors: bool (default True) - Continue on row errors + + Returns: + Import results with counts and any errors + + Requires permission: users.import + """ + job = db.query(ImportJob).filter(ImportJob.id == job_id).first() + if not job: + raise HTTPException(status_code=404, detail="Import job not found") + + if job.status != ImportJobStatus.preview_ready: + raise HTTPException( + status_code=400, + detail=f"Import job is not in preview_ready status (current: {job.status.value})" + ) + + if job.wordpress_metadata.get('import_type') != 'template': + raise HTTPException( + status_code=400, + detail="This is not a template-based import job" + ) + + # Update status + job.status = ImportJobStatus.processing + db.commit() + + # Get options + skip_notifications = options.get('skip_notifications', True) + update_existing = options.get('update_existing', False) + import_subscriptions = options.get('import_subscriptions', True) + import_donations = options.get('import_donations', True) + import_payments = options.get('import_payments', True) + import_registration_data = options.get('import_registration_data', True) + skip_errors = options.get('skip_errors', True) + + # Get data from job + full_data = job.wordpress_metadata.get('full_data', {}) + users_data = full_data.get('users', []) + subscriptions_data = full_data.get('subscriptions', []) if import_subscriptions else [] + donations_data = full_data.get('donations', []) if import_donations else [] + payments_data = full_data.get('payments', []) if import_payments else [] + registration_data = full_data.get('registration_data', []) if import_registration_data else [] + + # Track results + imported_user_ids = [] + user_email_to_id = {} + successful_rows = 0 + failed_rows = 0 + updated_rows = 0 + errors = [] + + # Get existing users by email for updates + existing_users = {} + if update_existing: + for user in db.query(User).filter( + User.email.in_([row.get('email', '').lower() for row in users_data if row.get('email')]) + ).all(): + existing_users[user.email.lower()] = user + + # Import users + for row in users_data: + try: + email = row.get('email', '').lower() + if not email: + continue + + # Check if user exists + existing_user = existing_users.get(email) + + if existing_user and not update_existing: + # Skip existing users if update not enabled + errors.append({ + 'row': row.get('_row_number', 0), + 'email': email, + 'error': 'User already exists (update_existing=False)' + }) + failed_rows += 1 + continue + + if existing_user and update_existing: + # Update existing user + user = existing_user + updated_rows += 1 + else: + # Create new user + user = User( + email=email, + password_hash=get_password_hash(secrets.token_urlsafe(32)), # Random password + email_verified=True, + force_password_change=True, + import_source='template_import', + import_job_id=job.id, + ) + db.add(user) + + # Set user fields from row data + if row.get('first_name'): + user.first_name = row['first_name'] + if row.get('last_name'): + user.last_name = row['last_name'] + if row.get('phone'): + user.phone = row['phone'] + if row.get('address'): + user.address = row['address'] + if row.get('city'): + user.city = row['city'] + if row.get('state'): + user.state = row['state'] + if row.get('zipcode'): + user.zipcode = row['zipcode'] + + # Status and role + if row.get('status'): + try: + user.status = UserStatus[row['status']] + except KeyError: + user.status = UserStatus.active + else: + user.status = UserStatus.active + + if row.get('role'): + try: + user.role = UserRole[row['role']] + except KeyError: + user.role = UserRole.member + else: + user.role = UserRole.member + + # Dates + if row.get('date_of_birth'): + user.date_of_birth = row['date_of_birth'] + if row.get('member_since'): + user.member_since = row['member_since'] + + # Partner info + if row.get('partner_first_name'): + user.partner_first_name = row['partner_first_name'] + if row.get('partner_last_name'): + user.partner_last_name = row['partner_last_name'] + if row.get('partner_is_member') is not None: + user.partner_is_member = row['partner_is_member'] + + # Referral and lead sources + if row.get('referred_by_member_name'): + user.referred_by_member_name = row['referred_by_member_name'] + if row.get('lead_sources'): + user.lead_sources = row['lead_sources'] if isinstance(row['lead_sources'], list) else [row['lead_sources']] + + # Directory settings + if row.get('show_in_directory') is not None: + user.show_in_directory = row['show_in_directory'] + if row.get('directory_bio'): + user.directory_bio = row['directory_bio'] + if row.get('directory_email') is not None: + user.directory_email = row['directory_email'] + if row.get('directory_phone') is not None: + user.directory_phone = row['directory_phone'] + if row.get('directory_address') is not None: + user.directory_address = row['directory_address'] + + # Store directory_display_name in custom_registration_data + if row.get('directory_display_name'): + custom_data = user.custom_registration_data or {} + custom_data['directory_display_name'] = row['directory_display_name'] + user.custom_registration_data = custom_data + + # Newsletter preferences + if row.get('newsletter_subscribed') is not None: + user.newsletter_subscribed = row['newsletter_subscribed'] + if row.get('newsletter_publish_name') is not None: + user.newsletter_publish_name = row['newsletter_publish_name'] + if row.get('newsletter_publish_photo') is not None: + user.newsletter_publish_photo = row['newsletter_publish_photo'] + if row.get('newsletter_publish_birthday') is not None: + user.newsletter_publish_birthday = row['newsletter_publish_birthday'] + + # Volunteer interests + if row.get('volunteer_interests'): + user.volunteer_interests = row['volunteer_interests'] if isinstance(row['volunteer_interests'], list) else [row['volunteer_interests']] + + # Scholarship + if row.get('scholarship_requested') is not None: + user.scholarship_requested = row['scholarship_requested'] + if row.get('scholarship_reason'): + user.scholarship_reason = row['scholarship_reason'] + + # Social media + if row.get('social_media_facebook'): + user.social_media_facebook = row['social_media_facebook'] + if row.get('social_media_instagram'): + user.social_media_instagram = row['social_media_instagram'] + if row.get('social_media_twitter'): + user.social_media_twitter = row['social_media_twitter'] + if row.get('social_media_linkedin'): + user.social_media_linkedin = row['social_media_linkedin'] + + db.flush() # Get user ID + imported_user_ids.append(str(user.id)) + user_email_to_id[email] = user.id + successful_rows += 1 + + except Exception as e: + logger.error(f"Error importing user row {row.get('_row_number', 0)}: {str(e)}") + errors.append({ + 'row': row.get('_row_number', 0), + 'email': row.get('email', ''), + 'error': str(e) + }) + failed_rows += 1 + if not skip_errors: + db.rollback() + job.status = ImportJobStatus.failed + job.errors = errors + db.commit() + raise HTTPException(status_code=500, detail=f"Import failed at row {row.get('_row_number', 0)}: {str(e)}") + continue + + # Import subscriptions + subscriptions_imported = 0 + for row in subscriptions_data: + try: + email = row.get('email', '').lower() + user_id = user_email_to_id.get(email) + + if not user_id: + # Try to find existing user + existing = db.query(User).filter(User.email == email).first() + if existing: + user_id = existing.id + + if not user_id: + continue # Skip orphaned subscription + + # Find or create plan + plan_name = row.get('plan_name', 'Imported Plan') + plan = db.query(SubscriptionPlan).filter(SubscriptionPlan.name == plan_name).first() + + if not plan: + # Create a basic plan for imported subscriptions + plan = SubscriptionPlan( + name=plan_name, + description=f"Imported plan: {plan_name}", + price=row.get('amount', 0) or 0, + duration_months=12, + is_active=True, + ) + db.add(plan) + db.flush() + + # Convert amount to cents if provided + amount_cents = None + if row.get('amount'): + try: + amount_cents = int(float(row['amount']) * 100) + except (ValueError, TypeError): + amount_cents = None + + # Create subscription record with Stripe fields + subscription = Subscription( + user_id=user_id, + plan_id=plan.id, + status=SubscriptionStatus[row.get('status', 'active')] if row.get('status') else SubscriptionStatus.active, + amount_paid_cents=amount_cents, + base_subscription_cents=amount_cents or 0, + donation_cents=0, + start_date=row.get('start_date') or datetime.now(timezone.utc), + end_date=row.get('end_date'), + # Stripe transaction fields + stripe_subscription_id=row.get('stripe_subscription_id'), + stripe_customer_id=row.get('stripe_customer_id'), + stripe_payment_intent_id=row.get('stripe_payment_intent_id'), + stripe_invoice_id=row.get('stripe_invoice_id'), + stripe_charge_id=row.get('stripe_charge_id'), + stripe_receipt_url=row.get('stripe_receipt_url'), + card_last4=row.get('card_last4'), + card_brand=row.get('card_brand'), + payment_method=row.get('payment_method'), + ) + db.add(subscription) + subscriptions_imported += 1 + + except Exception as e: + logger.error(f"Error importing subscription: {str(e)}") + errors.append({ + 'type': 'subscription', + 'row': row.get('_row_number', 0), + 'email': row.get('email', ''), + 'error': str(e) + }) + + # Import donations + donations_imported = 0 + for row in donations_data: + try: + email = row.get('email', '').lower() if row.get('email') else None + user_id = user_email_to_id.get(email) if email else None + + if not user_id and email: + existing = db.query(User).filter(User.email == email).first() + if existing: + user_id = existing.id + + # Convert amount to cents + amount_cents = 0 + if row.get('amount'): + try: + amount_cents = int(float(row['amount']) * 100) + except (ValueError, TypeError): + amount_cents = 0 + + # Parse payment completed date if provided + payment_completed_at = None + if isinstance(row.get('date'), date): + payment_completed_at = datetime.combine(row['date'], datetime.min.time()).replace(tzinfo=timezone.utc) + + donation = Donation( + user_id=user_id, + donor_name=row.get('donor_name') if not user_id else None, + donor_email=email if not user_id else None, + amount_cents=amount_cents, + donation_type=DonationType[row.get('type', 'member')] if row.get('type') else DonationType.member, + status=DonationStatus[row.get('status', 'completed')] if row.get('status') else DonationStatus.completed, + created_at=payment_completed_at or datetime.now(timezone.utc), + # Stripe transaction fields + stripe_payment_intent_id=row.get('stripe_payment_intent_id'), + stripe_charge_id=row.get('stripe_charge_id'), + stripe_customer_id=row.get('stripe_customer_id'), + stripe_checkout_session_id=row.get('stripe_checkout_session_id'), + stripe_receipt_url=row.get('stripe_receipt_url'), + card_last4=row.get('card_last4'), + card_brand=row.get('card_brand'), + payment_method=row.get('payment_method'), + payment_completed_at=payment_completed_at, + notes=row.get('notes'), + ) + db.add(donation) + donations_imported += 1 + + except Exception as e: + logger.error(f"Error importing donation: {str(e)}") + errors.append({ + 'type': 'donation', + 'row': row.get('_row_number', 0), + 'error': str(e) + }) + + # Import registration data as custom_registration_data JSON + registration_fields_imported = 0 + for row in registration_data: + try: + email = row.get('email', '').lower() + user_id = user_email_to_id.get(email) + + if not user_id: + existing = db.query(User).filter(User.email == email).first() + if existing: + user_id = existing.id + + if not user_id: + continue + + user = db.query(User).filter(User.id == user_id).first() + if user: + custom_data = user.custom_registration_data or {} + field_name = row.get('field_name') + field_value = row.get('field_value') + if field_name: + custom_data[field_name] = field_value + user.custom_registration_data = custom_data + registration_fields_imported += 1 + + except Exception as e: + logger.error(f"Error importing registration data: {str(e)}") + errors.append({ + 'type': 'registration_data', + 'row': row.get('_row_number', 0), + 'error': str(e) + }) + + # Update job status + job.processed_rows = len(users_data) + job.successful_rows = successful_rows + job.failed_rows = failed_rows + job.imported_user_ids = imported_user_ids + job.status = ImportJobStatus.completed if failed_rows == 0 else ImportJobStatus.partial + job.completed_at = datetime.now(timezone.utc) + job.errors = errors + + db.commit() + + logger.info( + f"Template import {job.id} completed: {successful_rows} users imported, " + f"{updated_rows} updated, {failed_rows} failed, " + f"{subscriptions_imported} subscriptions, {donations_imported} donations" + ) + + return { + "success": True, + "import_job_id": str(job.id), + "results": { + "users": { + "total": len(users_data), + "imported": successful_rows, + "updated": updated_rows, + "failed": failed_rows, + }, + "subscriptions": { + "total": len(subscriptions_data), + "imported": subscriptions_imported, + }, + "donations": { + "total": len(donations_data), + "imported": donations_imported, + }, + "registration_fields": { + "total": len(registration_data), + "imported": registration_fields_imported, + }, + }, + "errors": errors[:20], # Return first 20 errors + "error_count": len(errors), } # ============================================================================ -# WordPress CSV Import Endpoints +# WordPress CSV Import Endpoints (Legacy) # ============================================================================ @api_router.post("/admin/import/upload-csv") @@ -4577,6 +5524,582 @@ async def upload_wordpress_csv( os.unlink(tmp_path) +# ============================================================================ +# Comprehensive Multi-File Import (WordPress Users + Members + Payments) +# NOTE: These routes MUST be defined BEFORE the generic /admin/import/{job_id} routes +# to avoid FastAPI matching "comprehensive" as a job_id parameter +# ============================================================================ + +@api_router.post("/admin/import/comprehensive/upload") +async def upload_comprehensive_import( + users_file: UploadFile = File(..., description="WordPress users export CSV (required)"), + members_file: Optional[UploadFile] = File(None, description="PMS members export CSV (optional)"), + payments_file: Optional[UploadFile] = File(None, description="PMS payments export CSV (optional)"), + current_user: User = Depends(require_permission("users.import")), + db: Session = Depends(get_db) +): + """ + Upload multiple CSV files for comprehensive WordPress import. + + This endpoint accepts: + - users_file: WordPress users export (required) - contains full user profile data + - members_file: PMS members export (optional) - contains subscription data + - payments_file: PMS payments export (optional) - contains payment history + + Process: + 1. Save uploaded files temporarily + 2. Parse and validate all files + 3. Cross-reference data by email/user_id + 4. Generate preview with field mapping + 5. Create ImportJob for tracking + 6. Upload CSV files to R2 for persistent storage + + Returns: + Import job ID and comprehensive preview summary + + Requires permission: users.import + """ + from wordpress_parser import analyze_comprehensive_import + + tmp_paths = {} + file_contents = {} # Store raw bytes for R2 upload + r2_keys = {} # Store R2 object keys + + try: + # Validate users file (required) + if not users_file.filename.endswith('.csv'): + raise HTTPException(status_code=400, detail="Users file must be a CSV") + + # Get existing emails for duplicate detection + existing_emails = set( + email[0].lower() for email in db.query(User.email).all() + ) + + # Generate a unique job ID for R2 folder + job_uuid = str(uuid.uuid4()) + r2_folder = f"imports/{job_uuid}" + + # Read and save users file to temp (for analysis) + users_content = await users_file.read() + file_contents['users'] = (users_content, users_file.filename) + users_tmp = tempfile.NamedTemporaryFile(delete=False, suffix='.csv') + users_tmp.write(users_content) + users_tmp.close() + tmp_paths['users'] = users_tmp.name + + # Read and save members file if provided + members_tmp_path = None + if members_file and members_file.filename: + if not members_file.filename.endswith('.csv'): + raise HTTPException(status_code=400, detail="Members file must be a CSV") + members_content = await members_file.read() + file_contents['members'] = (members_content, members_file.filename) + members_tmp = tempfile.NamedTemporaryFile(delete=False, suffix='.csv') + members_tmp.write(members_content) + members_tmp.close() + tmp_paths['members'] = members_tmp.name + members_tmp_path = members_tmp.name + + # Read and save payments file if provided + payments_tmp_path = None + if payments_file and payments_file.filename: + if not payments_file.filename.endswith('.csv'): + raise HTTPException(status_code=400, detail="Payments file must be a CSV") + payments_content = await payments_file.read() + file_contents['payments'] = (payments_content, payments_file.filename) + payments_tmp = tempfile.NamedTemporaryFile(delete=False, suffix='.csv') + payments_tmp.write(payments_content) + payments_tmp.close() + tmp_paths['payments'] = payments_tmp.name + payments_tmp_path = payments_tmp.name + + # Analyze all files + analysis = analyze_comprehensive_import( + users_csv_path=tmp_paths['users'], + members_csv_path=members_tmp_path, + payments_csv_path=payments_tmp_path, + existing_emails=existing_emails + ) + + # Upload CSV files to R2 for persistent storage + r2_storage = get_r2_storage() + for file_type, (content, filename) in file_contents.items(): + _, r2_key, _ = await r2_storage.upload_bytes( + content=content, + folder=r2_folder, + filename=f"{file_type}_{filename}", + content_type='text/csv' + ) + r2_keys[file_type] = r2_key + logger.info(f"Uploaded {file_type} CSV to R2: {r2_key}") + + # Create import job - serialize all data to ensure JSON compatibility + wordpress_metadata = serialize_for_json({ + 'comprehensive': True, + 'files_uploaded': { + 'users': users_file.filename, + 'members': members_file.filename if members_file else None, + 'payments': payments_file.filename if payments_file else None + }, + 'r2_keys': r2_keys, # Store R2 keys for later retrieval + 'preview_data': analysis['users']['preview'], + 'members_data': analysis['members'].get('data', {}), + 'payments_data': analysis['payments'].get('data', {}), + 'summary': analysis['summary'], + 'data_quality': analysis['users'].get('data_quality', {}) + }) + + import_job = ImportJob( + id=uuid.UUID(job_uuid), # Use the same UUID we used for R2 folder + filename=users_file.filename, + total_rows=analysis['summary']['total_users'], + status=ImportJobStatus.preview_ready, + imported_by=current_user.id, + wordpress_metadata=wordpress_metadata + ) + + db.add(import_job) + db.commit() + db.refresh(import_job) + + logger.info(f"Comprehensive import uploaded: {import_job.id} - {analysis['summary']['total_users']} users by {current_user.email}") + + return { + 'import_job_id': str(import_job.id), + 'summary': analysis['summary'], + 'users': { + 'total': analysis['users']['total'], + 'valid': analysis['users']['valid'], + 'warnings': analysis['users']['warnings'], + 'errors': analysis['users']['errors'] + }, + 'members': { + 'total': analysis['members']['total'], + 'matched': analysis['members']['matched'], + 'unmatched': analysis['members']['unmatched'] + }, + 'payments': { + 'total': analysis['payments']['total'], + 'matched': analysis['payments']['matched'], + 'total_amount_cents': analysis['payments']['total_amount_cents'] + } + } + + except HTTPException: + raise + except Exception as e: + logger.error(f"Failed to upload comprehensive import: {str(e)}") + # Clean up R2 files if upload failed after partial upload + if r2_keys: + try: + r2_storage = get_r2_storage() + await r2_storage.delete_multiple(list(r2_keys.values())) + except Exception: + pass # Best effort cleanup + raise HTTPException(status_code=500, detail=f"Failed to process CSV files: {str(e)}") + finally: + # Clean up temp files + for path in tmp_paths.values(): + if os.path.exists(path): + os.unlink(path) + + +@api_router.get("/admin/import/comprehensive/{job_id}/preview") +async def get_comprehensive_import_preview( + job_id: str, + page: int = 1, + page_size: int = 50, + filter_errors: bool = False, + filter_warnings: bool = False, + current_user: User = Depends(require_permission("users.view")), + db: Session = Depends(get_db) +): + """ + Get paginated preview for comprehensive import with cross-referenced data. + + Args: + job_id: Import job UUID + page: Page number (1-indexed) + page_size: Rows per page (default 50) + filter_errors: Show only rows with errors + filter_warnings: Show only rows with warnings + + Returns: + Paginated preview with user data, subscription info, and payment history + + Requires permission: users.view + """ + job = db.query(ImportJob).filter(ImportJob.id == job_id).first() + if not job: + raise HTTPException(status_code=404, detail="Import job not found") + + if job.status != ImportJobStatus.preview_ready: + raise HTTPException( + status_code=400, + detail=f"Import job is not in preview_ready status (current: {job.status.value})" + ) + + # Get preview data + preview_data = job.wordpress_metadata.get('preview_data', []) + + # Apply filters + if filter_errors: + preview_data = [row for row in preview_data if row.get('errors')] + if filter_warnings: + preview_data = [row for row in preview_data if row.get('warnings')] + + # Paginate + total_pages = (len(preview_data) + page_size - 1) // page_size + start_idx = (page - 1) * page_size + end_idx = start_idx + page_size + + return { + 'page': page, + 'page_size': page_size, + 'total_pages': total_pages, + 'total_rows': len(preview_data), + 'rows': preview_data[start_idx:end_idx], + 'summary': job.wordpress_metadata.get('summary', {}), + 'files_uploaded': job.wordpress_metadata.get('files_uploaded', {}) + } + + +@api_router.post("/admin/import/comprehensive/{job_id}/execute") +async def execute_comprehensive_import( + job_id: str, + overrides: dict = {}, + options: dict = {}, + current_user: User = Depends(require_permission("users.import")), + db: Session = Depends(get_db) +): + """ + Execute comprehensive import with full data including subscriptions. + + Options: + - send_welcome_emails: bool (default False) - Send welcome emails + - send_password_emails: bool (default False) - Send password reset emails + - skip_notifications: bool (default True) - Skip all email notifications + - import_subscriptions: bool (default True) - Create subscription records + - import_payment_history: bool (default True) - Import payment records as notes + - skip_errors: bool (default True) - Continue on row errors + + Overrides: + Dict mapping row_number to overrides: + - status: Override suggested status + - role: Override suggested role + - skip: bool - Skip this row entirely + + Returns: + Import results with counts and any errors + + Requires permission: users.import + """ + job = db.query(ImportJob).filter(ImportJob.id == job_id).first() + if not job: + raise HTTPException(status_code=404, detail="Import job not found") + + if job.status != ImportJobStatus.preview_ready: + raise HTTPException( + status_code=400, + detail=f"Import job is not in preview_ready status (current: {job.status.value})" + ) + + # Update status + job.status = ImportJobStatus.processing + db.commit() + + # Get data from job metadata + preview_data = job.wordpress_metadata.get('preview_data', []) + members_data = job.wordpress_metadata.get('members_data', {}) + payments_data = job.wordpress_metadata.get('payments_data', {}) + + # Import options + skip_notifications = options.get('skip_notifications', True) + send_welcome_emails = options.get('send_welcome_emails', False) and not skip_notifications + send_password_emails = options.get('send_password_emails', False) and not skip_notifications + import_subscriptions = options.get('import_subscriptions', True) + import_payment_history = options.get('import_payment_history', True) + skip_errors = options.get('skip_errors', True) + + # Results tracking + imported_user_ids = [] + created_subscriptions = 0 + successful_rows = 0 + failed_rows = 0 + skipped_rows = 0 + errors = [] + + # Default password hash + default_password_hash = get_password_hash(secrets.token_urlsafe(32)) + + # Get or create default subscription plan for imports + default_plan = db.query(SubscriptionPlan).filter(SubscriptionPlan.active == True).first() + + try: + for idx, row_data in enumerate(preview_data): + row_num = row_data['row_number'] + + # Check for skip override + if str(row_num) in overrides and overrides[str(row_num)].get('skip'): + skipped_rows += 1 + continue + + try: + # Skip rows with critical errors + if row_data.get('errors') and skip_errors: + failed_rows += 1 + errors.append({ + 'row': row_num, + 'email': row_data.get('email'), + 'error': ', '.join(row_data['errors']) + }) + continue + + email = row_data.get('email', '').lower() + if not email: + failed_rows += 1 + errors.append({'row': row_num, 'email': '', 'error': 'Missing email'}) + continue + + # Check existing user + existing = db.query(User).filter(User.email == email).first() + if existing: + failed_rows += 1 + errors.append({'row': row_num, 'email': email, 'error': 'Email already exists'}) + continue + + # Apply overrides + final_status = row_data['suggested_status'] + final_role = row_data['suggested_role'] + if str(row_num) in overrides: + final_status = overrides[str(row_num)].get('status', final_status) + final_role = overrides[str(row_num)].get('role', final_role) + + # Get full user data from row + user_data = row_data.get('user_data', {}) + custom_data = row_data.get('custom_data', {}) + newsletter_prefs = row_data.get('newsletter_prefs', {}) + + # Parse date_of_birth if string + dob = user_data.get('date_of_birth') + if isinstance(dob, str): + try: + dob = datetime.fromisoformat(dob.replace('Z', '+00:00')) + except: + dob = None + + # Parse member_since if string + member_since = user_data.get('member_since') + if isinstance(member_since, str): + try: + member_since = datetime.fromisoformat(member_since.replace('Z', '+00:00')) + except: + member_since = None + + # Create user with all mapped fields + new_user = User( + email=email, + password_hash=default_password_hash, + first_name=user_data.get('first_name', ''), + last_name=user_data.get('last_name', ''), + phone=user_data.get('phone', '0000000000'), + address=user_data.get('address', ''), + city=user_data.get('city', ''), + state=user_data.get('state', ''), + zipcode=user_data.get('zipcode', ''), + date_of_birth=dob or datetime(1900, 1, 1), + status=UserStatus[final_status], + role=UserRole[final_role], + + # Partner info + partner_first_name=user_data.get('partner_first_name'), + partner_last_name=user_data.get('partner_last_name'), + partner_is_member=user_data.get('partner_is_member', False), + partner_plan_to_become_member=user_data.get('partner_plan_to_become_member', False), + + # Referral + referred_by_member_name=user_data.get('referred_by_member_name'), + lead_sources=user_data.get('lead_sources', []), + + # Newsletter + newsletter_subscribed=user_data.get('newsletter_subscribed', False), + newsletter_publish_name=newsletter_prefs.get('newsletter_publish_name', False), + newsletter_publish_photo=newsletter_prefs.get('newsletter_publish_photo', False), + newsletter_publish_birthday=newsletter_prefs.get('newsletter_publish_birthday', False), + newsletter_publish_none=newsletter_prefs.get('newsletter_publish_none', False), + + # Volunteer + volunteer_interests=user_data.get('volunteer_interests', []), + + # Scholarship + scholarship_requested=user_data.get('scholarship_requested', False), + scholarship_reason=user_data.get('scholarship_reason'), + + # Directory + show_in_directory=user_data.get('show_in_directory', False), + directory_email=user_data.get('directory_email'), + directory_bio=user_data.get('directory_bio'), + directory_address=user_data.get('directory_address'), + directory_phone=user_data.get('directory_phone'), + directory_partner_name=user_data.get('directory_partner_name'), + profile_photo_url=user_data.get('profile_photo_url'), + + # Social media + social_media_facebook=user_data.get('social_media_facebook'), + social_media_instagram=user_data.get('social_media_instagram'), + social_media_twitter=user_data.get('social_media_twitter'), + social_media_linkedin=user_data.get('social_media_linkedin'), + + # Stripe + stripe_customer_id=user_data.get('stripe_customer_id'), + + # Custom registration data + custom_registration_data=custom_data if custom_data else {}, + + # Metadata + member_since=member_since, + email_verified=True, + force_password_change=True, + accepts_tos=True, + tos_accepted_at=datetime.now(timezone.utc), + import_source='wordpress', + import_job_id=job.id, + wordpress_user_id=user_data.get('wordpress_user_id'), + wordpress_registered_date=user_data.get('wordpress_registered_date') + ) + + db.add(new_user) + db.flush() + imported_user_ids.append(str(new_user.id)) + + # Create subscription if data available + subscription_info = members_data.get(email) + if import_subscriptions and subscription_info and default_plan: + # Parse dates from ISO strings + start_date = subscription_info.get('start_date') + end_date = subscription_info.get('end_date') + + if isinstance(start_date, str): + try: + start_date = datetime.fromisoformat(start_date.replace('Z', '+00:00')) + except: + start_date = datetime.now(timezone.utc) + + if isinstance(end_date, str): + try: + end_date = datetime.fromisoformat(end_date.replace('Z', '+00:00')) + except: + end_date = None + + # Build payment history notes + payment_notes = "" + if import_payment_history: + user_payments = payments_data.get(email, []) + if user_payments: + payment_notes = "WordPress Payment History:\n" + for p in user_payments: + p_date = p.get('date', 'Unknown date') + p_amount = p.get('amount_cents', 0) / 100 + p_status = p.get('status', 'unknown') + payment_notes += f"- ${p_amount:.2f} on {p_date} ({p_status})\n" + + # Map subscription status + sub_status_str = subscription_info.get('status', 'active') + try: + sub_status = SubscriptionStatus[sub_status_str] + except KeyError: + sub_status = SubscriptionStatus.active + + subscription = Subscription( + user_id=new_user.id, + plan_id=default_plan.id, + status=sub_status, + start_date=start_date or datetime.now(timezone.utc), + end_date=end_date, + amount_paid_cents=default_plan.minimum_price_cents, + base_subscription_cents=default_plan.minimum_price_cents, + donation_cents=0, + manual_payment=True, + manual_payment_notes=f"Imported from WordPress. {payment_notes}".strip(), + manual_payment_admin_id=current_user.id, + manual_payment_date=start_date, + payment_method=subscription_info.get('payment_method', 'manual') + ) + db.add(subscription) + created_subscriptions += 1 + + successful_rows += 1 + + # Commit in batches + if (idx + 1) % 20 == 0: + db.commit() + job.processed_rows = idx + 1 + db.commit() + + except Exception as e: + logger.error(f"Failed to import row {row_num}: {str(e)}") + failed_rows += 1 + errors.append({'row': row_num, 'email': row_data.get('email', ''), 'error': str(e)}) + if not skip_errors: + db.rollback() + raise HTTPException(status_code=500, detail=f"Import failed at row {row_num}: {str(e)}") + + # Final commit + db.commit() + + # Update job + job.processed_rows = len(preview_data) + job.successful_rows = successful_rows + job.failed_rows = failed_rows + job.status = ImportJobStatus.completed if failed_rows == 0 else ImportJobStatus.partial + job.imported_user_ids = imported_user_ids + job.errors = errors + job.completed_at = datetime.now(timezone.utc) + db.commit() + + # Send emails if requested + emails_sent = 0 + if send_password_emails and imported_user_ids: + for user_id_str in imported_user_ids[:100]: # Limit to 100 to avoid timeouts + try: + user_uuid = uuid.UUID(user_id_str) + user = db.query(User).filter(User.id == user_uuid).first() + if user: + reset_token = create_password_reset_token(user.email) + reset_url = f"{os.getenv('FRONTEND_URL')}/reset-password?token={reset_token}" + await send_password_reset_email(user.email, user.first_name, reset_url) + emails_sent += 1 + except Exception as e: + logger.warning(f"Failed to send email to {user_id_str}: {str(e)}") + + logger.info(f"Comprehensive import executed: {job.id} - {successful_rows}/{len(preview_data)} users, {created_subscriptions} subscriptions by {current_user.email}") + + return { + 'successful_rows': successful_rows, + 'failed_rows': failed_rows, + 'skipped_rows': skipped_rows, + 'created_subscriptions': created_subscriptions, + 'imported_user_ids': imported_user_ids, + 'emails_sent': emails_sent, + 'errors': errors[:50] # Limit error response size + } + + except HTTPException: + raise + except Exception as e: + db.rollback() + job.status = ImportJobStatus.failed + job.errors = [{'error': str(e)}] + db.commit() + logger.error(f"Comprehensive import failed: {str(e)}") + raise HTTPException(status_code=500, detail=f"Import failed: {str(e)}") + + +# ============================================================================ +# Generic Import Endpoints (legacy WordPress single-file import) +# NOTE: These routes MUST come AFTER the comprehensive routes above +# ============================================================================ + @api_router.get("/admin/import/{job_id}/preview") async def get_import_preview( job_id: str, @@ -9385,18 +10908,28 @@ print(f"✓ Security headers configured (Production: {IS_PRODUCTION})") # CORS Configuration (Added second, executes first) cors_origins = os.environ.get('CORS_ORIGINS', '') + +# Default development origins +DEFAULT_DEV_ORIGINS = [ + "http://localhost:3000", + "http://localhost:8000", + "http://127.0.0.1:3000", + "http://127.0.0.1:8000", +] + if cors_origins: - # Use explicitly configured origins - allowed_origins = [origin.strip() for origin in cors_origins.split(',')] + if cors_origins.strip() == '*': + # '*' doesn't work with credentials, so expand to common dev origins + # For true wildcard in production, set specific origins instead + allowed_origins = DEFAULT_DEV_ORIGINS + print("⚠️ CORS_ORIGINS='*' expanded to dev origins (can't use * with credentials)") + else: + # Use explicitly configured origins + allowed_origins = [origin.strip() for origin in cors_origins.split(',')] else: # Default to common development origins if not configured - allowed_origins = [ - "http://localhost:3000", - "http://localhost:8000", - "http://127.0.0.1:3000", - "http://127.0.0.1:8000" - ] - print(f"⚠️ WARNING: CORS_ORIGINS not set. Using defaults: {allowed_origins}") + allowed_origins = DEFAULT_DEV_ORIGINS + print(f"⚠️ WARNING: CORS_ORIGINS not set. Using defaults.") print("⚠️ For production, set CORS_ORIGINS in .env file!") print(f"✓ CORS allowed origins: {allowed_origins}") diff --git a/wordpress_parser.py b/wordpress_parser.py index 4a1e329..97f6407 100644 --- a/wordpress_parser.py +++ b/wordpress_parser.py @@ -10,21 +10,127 @@ Key Features: - Validate and standardize user data (DOB, phone numbers) - Generate smart status suggestions based on approval and subscription data - Comprehensive data quality analysis and error reporting +- Multi-file import support (Users, Members, Payments CSVs) +- Field mapping based on Meta Name Reference document Author: Claude Code Date: 2025-12-24 +Updated: 2026-02-03 - Added comprehensive multi-file import support """ import csv import re import logging from datetime import datetime -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple, Any import phpserialize +import pandas as pd logger = logging.getLogger(__name__) +# ============================================================================ +# Meta Name Reference Field Mapping (from client's WordPress export) +# ============================================================================ + +# Maps WordPress meta names to our database fields +# Format: 'wordpress_meta_name': ('db_field', 'field_type', 'parser_function') +META_FIELD_MAPPING = { + # Basic user info + 'first_name': ('first_name', 'string', None), + 'last_name': ('last_name', 'string', None), + 'user_email': ('email', 'string', 'lowercase'), + 'user_login': ('username', 'string', None), # For reference only + 'address': ('address', 'string', None), + 'city': ('city', 'string', None), + 'state': ('state', 'string', None), + 'zipcode': ('zipcode', 'string', None), + 'cell_phone': ('phone', 'string', 'phone'), + 'date_of_birth': ('date_of_birth', 'date', 'date_mmddyyyy'), + + # Partner info + 'partner_first_name': ('partner_first_name', 'string', None), + 'partner_last_name': ('partner_last_name', 'string', None), + 'partner_membership_status': ('partner_is_member', 'boolean', 'yes_no'), + 'partner_membership_consideration': ('partner_plan_to_become_member', 'boolean', 'yes_no'), + + # Newsletter preferences + 'newsletter_consent': ('newsletter_subscribed', 'boolean', 'yes_no'), + 'newsletter_checklist': ('newsletter_preferences', 'multi_value', 'newsletter_checklist'), + + # Referral and lead sources + 'member_referral': ('referred_by_member_name', 'string', None), + 'referral_source': ('lead_sources', 'multi_value', 'lead_sources'), + + # Volunteer interests + 'volunteer_checklist': ('volunteer_interests', 'multi_value', 'volunteer_checklist'), + + # Scholarship + 'scholarship_request': ('scholarship_requested', 'boolean', 'yes_no'), + 'scholarship_reason': ('scholarship_reason', 'string', None), + + # Directory settings + 'members_directory_filter': ('show_in_directory', 'boolean', 'yes_no'), + 'md_display_name': ('custom_registration_data.directory_display_name', 'custom', None), + 'md_email': ('directory_email', 'string', None), + 'description': ('directory_bio', 'string', None), + 'md_adress': ('directory_address', 'string', None), # Note: typo in WordPress + 'md_phone': ('directory_phone', 'string', None), + 'md_dob': ('directory_dob', 'date', 'date_mmddyyyy'), + 'md_partner_name': ('directory_partner_name', 'string', None), + 'md_avatar': ('profile_photo_url', 'string', None), + + # Metadata + 'member_since': ('member_since', 'date', 'date_various'), + 'user_registered': ('wordpress_registered_date', 'datetime', 'datetime_mysql'), + 'ID': ('wordpress_user_id', 'integer', None), + + # Stripe info (from WordPress) + 'pms_stripe_customer_id': ('stripe_customer_id', 'string', None), +} + +# Newsletter checklist option mapping +NEWSLETTER_CHECKLIST_OPTIONS = { + 'name': 'newsletter_publish_name', + 'photo': 'newsletter_publish_photo', + 'birthday': 'newsletter_publish_birthday', + 'none': 'newsletter_publish_none', + # Handle various WordPress stored formats + 'my name': 'newsletter_publish_name', + 'my photo': 'newsletter_publish_photo', + 'my birthday': 'newsletter_publish_birthday', +} + +# Volunteer interests mapping (WordPress values to our format) +VOLUNTEER_INTERESTS_MAP = { + 'events': 'Events', + 'fundraising': 'Fundraising', + 'communications': 'Communications', + 'membership': 'Membership', + 'board': 'Board of Directors', + 'other': 'Other', + # Handle various WordPress formats + 'help with events': 'Events', + 'help with fundraising': 'Fundraising', + 'help with communications': 'Communications', + 'help with membership': 'Membership', + 'serve on the board': 'Board of Directors', +} + +# Lead sources mapping +LEAD_SOURCES_MAP = { + 'current member': 'Current member', + 'friend': 'Friend', + 'outsmart magazine': 'OutSmart Magazine', + 'outsmart': 'OutSmart Magazine', + 'search engine': 'Search engine (Google etc.)', + 'google': 'Search engine (Google etc.)', + 'known about loaf': "I've known about LOAF for a long time", + 'long time': "I've known about LOAF for a long time", + 'other': 'Other', +} + + # ============================================================================ # WordPress Role Mapping Configuration # ============================================================================ @@ -283,6 +389,622 @@ def validate_dob(dob_str: str) -> Tuple[Optional[datetime], Optional[str]]: return None, f'Invalid date format: {dob_str} (expected MM/DD/YYYY)' +# ============================================================================ +# Enhanced Field Parsers for Meta Name Reference +# ============================================================================ + +def parse_boolean_yes_no(value: Any) -> bool: + """ + Parse yes/no style boolean values from WordPress. + + Handles: yes, no, true, false, 1, 0, checked, unchecked + """ + if value is None or (isinstance(value, float) and pd.isna(value)): + return False + + str_val = str(value).lower().strip() + return str_val in ('yes', 'true', '1', 'checked', 'on', 'y') + + +def parse_date_various(date_str: Any) -> Optional[datetime]: + """ + Parse dates in various formats commonly found in WordPress exports. + + Handles: + - MM/DD/YYYY (US format) + - YYYY-MM-DD (ISO format) + - DD/MM/YYYY (EU format - attempted if US fails) + - Month DD, YYYY (e.g., "January 15, 2020") + """ + if date_str is None or (isinstance(date_str, float) and pd.isna(date_str)): + return None + + date_str = str(date_str).strip() + if not date_str or date_str.lower() == 'nan': + return None + + # Try various formats + formats = [ + '%m/%d/%Y', # US: 01/15/2020 + '%Y-%m-%d', # ISO: 2020-01-15 + '%d/%m/%Y', # EU: 15/01/2020 + '%B %d, %Y', # Full: January 15, 2020 + '%b %d, %Y', # Short: Jan 15, 2020 + '%Y-%m-%d %H:%M:%S', # MySQL datetime + '%m/%Y', # Month/Year: 01/2020 + '%m-%Y', # Month-Year: 01-2020 + '%b-%Y', # Short month-Year: Jan-2020 + '%B-%Y', # Full month-Year: January-2020 + ] + + for fmt in formats: + try: + parsed = datetime.strptime(date_str, fmt) + # Validate year range + if 1900 <= parsed.year <= datetime.now().year + 1: + return parsed + except ValueError: + continue + + # Only log warning for strings that look like dates + if date_str and len(date_str) > 3: + logger.debug(f"Could not parse date: {date_str}") + return None + + +def parse_datetime_mysql(dt_str: Any) -> Optional[datetime]: + """Parse MySQL datetime format: YYYY-MM-DD HH:MM:SS""" + if dt_str is None or (isinstance(dt_str, float) and pd.isna(dt_str)): + return None + + try: + return datetime.strptime(str(dt_str).strip(), '%Y-%m-%d %H:%M:%S') + except ValueError: + return parse_date_various(dt_str) + + +def parse_newsletter_checklist(value: Any) -> Dict[str, bool]: + """ + Parse newsletter checklist multi-value field. + + WordPress stores this as comma-separated or PHP serialized values. + Returns dict mapping to our newsletter_publish_* fields. + """ + result = { + 'newsletter_publish_name': False, + 'newsletter_publish_photo': False, + 'newsletter_publish_birthday': False, + 'newsletter_publish_none': False, + } + + if value is None or (isinstance(value, float) and pd.isna(value)): + return result + + str_val = str(value).lower().strip() + if not str_val or str_val == 'nan': + return result + + # Try PHP serialized first + if str_val.startswith('a:'): + try: + parsed = phpserialize.loads(str_val.encode('utf-8')) + if isinstance(parsed, dict): + for key in parsed.keys(): + key_str = key.decode('utf-8') if isinstance(key, bytes) else str(key) + key_lower = key_str.lower() + for match_key, field in NEWSLETTER_CHECKLIST_OPTIONS.items(): + if match_key in key_lower: + result[field] = True + return result + except Exception: + pass + + # Try comma-separated values + items = [item.strip().lower() for item in str_val.split(',')] + for item in items: + for match_key, field in NEWSLETTER_CHECKLIST_OPTIONS.items(): + if match_key in item: + result[field] = True + + return result + + +def parse_volunteer_checklist(value: Any) -> List[str]: + """ + Parse volunteer interests checklist. + + Returns list of standardized volunteer interest labels. + """ + if value is None or (isinstance(value, float) and pd.isna(value)): + return [] + + str_val = str(value).lower().strip() + if not str_val or str_val == 'nan': + return [] + + interests = [] + + # Try PHP serialized first + if str_val.startswith('a:'): + try: + parsed = phpserialize.loads(str_val.encode('utf-8')) + if isinstance(parsed, dict): + for key in parsed.keys(): + key_str = key.decode('utf-8') if isinstance(key, bytes) else str(key) + key_lower = key_str.lower() + for match_key, label in VOLUNTEER_INTERESTS_MAP.items(): + if match_key in key_lower and label not in interests: + interests.append(label) + return interests + except Exception: + pass + + # Try comma-separated values + items = [item.strip().lower() for item in str_val.split(',')] + for item in items: + for match_key, label in VOLUNTEER_INTERESTS_MAP.items(): + if match_key in item and label not in interests: + interests.append(label) + + return interests + + +def parse_lead_sources(value: Any) -> List[str]: + """ + Parse referral/lead sources field. + + Returns list of standardized lead source labels. + """ + if value is None or (isinstance(value, float) and pd.isna(value)): + return [] + + str_val = str(value).lower().strip() + if not str_val or str_val == 'nan': + return [] + + sources = [] + + # Try PHP serialized first + if str_val.startswith('a:'): + try: + parsed = phpserialize.loads(str_val.encode('utf-8')) + if isinstance(parsed, dict): + for key in parsed.keys(): + key_str = key.decode('utf-8') if isinstance(key, bytes) else str(key) + key_lower = key_str.lower() + for match_key, label in LEAD_SOURCES_MAP.items(): + if match_key in key_lower and label not in sources: + sources.append(label) + return sources + except Exception: + pass + + # Try comma-separated values + items = [item.strip().lower() for item in str_val.split(',')] + for item in items: + matched = False + for match_key, label in LEAD_SOURCES_MAP.items(): + if match_key in item and label not in sources: + sources.append(label) + matched = True + break + # If no match, add as "Other" with original value + if not matched and item: + sources.append('Other') + + return sources + + +def transform_csv_row_to_user_data(row: Dict[str, Any], existing_emails: set = None) -> Dict[str, Any]: + """ + Transform a CSV row to user data dictionary using Meta Name Reference mapping. + + Args: + row: Dictionary of CSV column values + existing_emails: Set of emails already in database (for duplicate check) + + Returns: + Dictionary with: + - user_data: Fields that map to User model + - custom_data: Fields for custom_registration_data JSON + - newsletter_prefs: Newsletter preference booleans + - warnings: List of warning messages + - errors: List of error messages + """ + user_data = {} + custom_data = {} + newsletter_prefs = {} + warnings = [] + errors = [] + + # Process each mapped field + for csv_field, (db_field, field_type, parser) in META_FIELD_MAPPING.items(): + value = row.get(csv_field) + + # Skip if no value + if value is None or (isinstance(value, float) and pd.isna(value)): + continue + + try: + # Parse based on field type + if field_type == 'string': + if parser == 'lowercase': + parsed_value = str(value).strip().lower() + elif parser == 'phone': + parsed_value = standardize_phone(value) + if parsed_value == '0000000000': + warnings.append(f'Invalid phone: {value}') + else: + parsed_value = str(value).strip() if value else None + + elif field_type == 'integer': + parsed_value = int(value) if value else None + + elif field_type == 'boolean': + parsed_value = parse_boolean_yes_no(value) + + elif field_type == 'date': + if parser == 'date_mmddyyyy': + parsed_value, warning = validate_dob(value) + if warning: + warnings.append(warning) + else: + parsed_value = parse_date_various(value) + + elif field_type == 'datetime': + parsed_value = parse_datetime_mysql(value) + + elif field_type == 'multi_value': + if parser == 'newsletter_checklist': + newsletter_prefs = parse_newsletter_checklist(value) + continue # Handled separately + elif parser == 'volunteer_checklist': + parsed_value = parse_volunteer_checklist(value) + elif parser == 'lead_sources': + parsed_value = parse_lead_sources(value) + else: + parsed_value = [str(value)] + + elif field_type == 'custom': + # Store in custom_registration_data + custom_field = db_field.replace('custom_registration_data.', '') + custom_data[custom_field] = str(value).strip() if value else None + continue + + else: + parsed_value = value + + # Store in appropriate location + if parsed_value is not None: + user_data[db_field] = parsed_value + + except Exception as e: + warnings.append(f'Error parsing {csv_field}: {str(e)}') + + # Check for required fields + if not user_data.get('email'): + errors.append('Missing email address') + elif existing_emails and user_data['email'] in existing_emails: + errors.append('Email already exists in database') + + if not user_data.get('first_name'): + warnings.append('Missing first name') + + if not user_data.get('last_name'): + warnings.append('Missing last name') + + return { + 'user_data': user_data, + 'custom_data': custom_data, + 'newsletter_prefs': newsletter_prefs, + 'warnings': warnings, + 'errors': errors + } + + +# ============================================================================ +# Members CSV Parser (Subscription Data) +# ============================================================================ + +def parse_members_csv(file_path: str) -> Dict[str, Any]: + """ + Parse WordPress PMS Members export CSV for subscription data. + + Args: + file_path: Path to pms-export-members CSV file + + Returns: + Dictionary mapping user_email to subscription data + """ + members_data = {} + + try: + df = pd.read_csv(file_path) + + for _, row in df.iterrows(): + email = str(row.get('user_email', '')).strip().lower() + if not email or email == 'nan': + continue + + # Parse subscription dates + start_date = parse_date_various(row.get('start_date')) + expiration_date = parse_date_various(row.get('expiration_date')) + + # Map subscription status + wp_status = str(row.get('status', '')).lower().strip() + if wp_status == 'active': + sub_status = 'active' + elif wp_status in ('expired', 'abandoned'): + sub_status = 'expired' + elif wp_status in ('canceled', 'cancelled'): + sub_status = 'cancelled' + else: + sub_status = 'active' # Default + + # Parse payment gateway + payment_gateway = str(row.get('payment_gateway', '')).lower().strip() + if 'stripe' in payment_gateway: + payment_method = 'stripe' + elif 'paypal' in payment_gateway: + payment_method = 'paypal' + elif payment_gateway in ('manual', 'admin', ''): + payment_method = 'manual' + else: + payment_method = payment_gateway or 'manual' + + members_data[email] = { + 'subscription_plan_id': row.get('subscription_plan_id'), + 'subscription_plan_name': row.get('subscription_plan_name'), + 'start_date': start_date, + 'end_date': expiration_date, + 'status': sub_status, + 'payment_method': payment_method, + 'wordpress_user_id': row.get('user_id'), + 'billing_first_name': row.get('billing_first_name'), + 'billing_last_name': row.get('billing_last_name'), + 'billing_address': row.get('billing_address'), + 'billing_city': row.get('billing_city'), + 'billing_state': row.get('billing_state'), + 'billing_zip': row.get('billing_zip'), + 'card_last4': row.get('billing_card_last4'), + } + + except Exception as e: + logger.error(f"Error parsing members CSV: {str(e)}") + raise + + return members_data + + +# ============================================================================ +# Payments CSV Parser (Payment History) +# ============================================================================ + +def parse_payments_csv(file_path: str) -> Dict[str, List[Dict]]: + """ + Parse WordPress PMS Payments export CSV for payment history. + + Args: + file_path: Path to pms-export-payments CSV file + + Returns: + Dictionary mapping user_email to list of payment records + """ + payments_data = {} + + try: + df = pd.read_csv(file_path) + + for _, row in df.iterrows(): + email = str(row.get('user_email', '')).strip().lower() + if not email or email == 'nan': + continue + + # Parse payment date + payment_date = parse_date_various(row.get('date')) + + # Parse amount (convert to cents) + amount_str = str(row.get('amount', '0')).replace('$', '').replace(',', '').strip() + try: + amount_cents = int(float(amount_str) * 100) + except (ValueError, TypeError): + amount_cents = 0 + + # Map payment status + wp_status = str(row.get('status', '')).lower().strip() + if wp_status == 'completed': + payment_status = 'completed' + elif wp_status in ('pending', 'processing'): + payment_status = 'pending' + elif wp_status in ('failed', 'refunded'): + payment_status = 'failed' + else: + payment_status = 'completed' # Default for historical data + + payment_record = { + 'payment_id': row.get('payment_id'), + 'amount_cents': amount_cents, + 'status': payment_status, + 'date': payment_date, + 'payment_gateway': row.get('payment_gateway'), + 'transaction_id': row.get('transaction_id'), + 'profile_id': row.get('profile_id'), + 'subscription_plan_id': row.get('subscription_plan_id'), + 'wordpress_user_id': row.get('user_id'), + } + + if email not in payments_data: + payments_data[email] = [] + payments_data[email].append(payment_record) + + except Exception as e: + logger.error(f"Error parsing payments CSV: {str(e)}") + raise + + return payments_data + + +# ============================================================================ +# Comprehensive Import Analysis +# ============================================================================ + +def analyze_comprehensive_import( + users_csv_path: str, + members_csv_path: Optional[str] = None, + payments_csv_path: Optional[str] = None, + existing_emails: Optional[set] = None +) -> Dict[str, Any]: + """ + Analyze all CSV files for comprehensive import with cross-referencing. + + Args: + users_csv_path: Path to WordPress users export CSV (required) + members_csv_path: Path to PMS members CSV (optional) + payments_csv_path: Path to PMS payments CSV (optional) + existing_emails: Set of emails already in database + + Returns: + Comprehensive analysis with preview data for all files + """ + if existing_emails is None: + existing_emails = set() + + result = { + 'users': {'total': 0, 'valid': 0, 'warnings': 0, 'errors': 0, 'preview': []}, + 'members': {'total': 0, 'matched': 0, 'unmatched': 0, 'data': {}}, + 'payments': {'total': 0, 'matched': 0, 'total_amount_cents': 0, 'data': {}}, + 'summary': { + 'total_users': 0, + 'importable_users': 0, + 'duplicate_emails': 0, + 'users_with_subscriptions': 0, + 'users_with_payments': 0, + 'total_payment_amount': 0, + } + } + + # Parse members CSV if provided + members_data = {} + if members_csv_path: + try: + members_data = parse_members_csv(members_csv_path) + result['members']['total'] = len(members_data) + result['members']['data'] = members_data + except Exception as e: + result['members']['error'] = str(e) + + # Parse payments CSV if provided + payments_data = {} + if payments_csv_path: + try: + payments_data = parse_payments_csv(payments_csv_path) + result['payments']['total'] = sum(len(p) for p in payments_data.values()) + result['payments']['data'] = payments_data + result['payments']['total_amount_cents'] = sum( + sum(p['amount_cents'] for p in payments) + for payments in payments_data.values() + ) + except Exception as e: + result['payments']['error'] = str(e) + + # Parse users CSV + try: + df = pd.read_csv(users_csv_path) + result['users']['total'] = len(df) + + seen_emails = set() + total_warnings = 0 + total_errors = 0 + + for idx, row in df.iterrows(): + row_dict = row.to_dict() + transformed = transform_csv_row_to_user_data(row_dict, existing_emails) + + email = transformed['user_data'].get('email', '').lower() + + # Check for CSV duplicates + if email in seen_emails: + transformed['errors'].append(f'Duplicate email in CSV') + elif email: + seen_emails.add(email) + + # Cross-reference with members data + subscription_data = members_data.get(email) + if subscription_data: + result['members']['matched'] += 1 + + # Cross-reference with payments data + payment_records = payments_data.get(email, []) + if payment_records: + result['payments']['matched'] += 1 + + # Parse WordPress roles for role/status suggestion + wp_capabilities = row.get('wp_capabilities', '') + wp_roles = parse_php_serialized(wp_capabilities) + loaf_role, role_status = map_wordpress_role(wp_roles) + + # Determine status + approval_status = str(row.get('wppb_approval_status', '')).strip() + has_subscription = 'pms_subscription_plan_63' in wp_roles or subscription_data is not None + + if role_status: + suggested_status = role_status + else: + suggested_status = suggest_status(approval_status, has_subscription, loaf_role) + + # Build preview row + preview_row = { + 'row_number': idx + 1, + 'email': email, + 'first_name': transformed['user_data'].get('first_name', ''), + 'last_name': transformed['user_data'].get('last_name', ''), + 'phone': transformed['user_data'].get('phone', ''), + 'date_of_birth': transformed['user_data'].get('date_of_birth').isoformat() if transformed['user_data'].get('date_of_birth') else None, + 'wordpress_user_id': transformed['user_data'].get('wordpress_user_id'), + 'wordpress_roles': wp_roles, + 'suggested_role': loaf_role, + 'suggested_status': suggested_status, + 'has_subscription': has_subscription, + 'subscription_data': subscription_data, + 'payment_count': len(payment_records), + 'total_paid_cents': sum(p['amount_cents'] for p in payment_records), + 'user_data': transformed['user_data'], + 'custom_data': transformed['custom_data'], + 'newsletter_prefs': transformed['newsletter_prefs'], + 'warnings': transformed['warnings'], + 'errors': transformed['errors'], + } + + result['users']['preview'].append(preview_row) + total_warnings += len(transformed['warnings']) + total_errors += len(transformed['errors']) + + if not transformed['errors']: + result['users']['valid'] += 1 + + result['users']['warnings'] = total_warnings + result['users']['errors'] = total_errors + + # Calculate unmatched members + user_emails = {p['email'] for p in result['users']['preview'] if p['email']} + result['members']['unmatched'] = len(set(members_data.keys()) - user_emails) + + # Summary stats + result['summary']['total_users'] = result['users']['total'] + result['summary']['importable_users'] = result['users']['valid'] + result['summary']['duplicate_emails'] = len(seen_emails & existing_emails) + result['summary']['users_with_subscriptions'] = result['members']['matched'] + result['summary']['users_with_payments'] = result['payments']['matched'] + result['summary']['total_payment_amount'] = result['payments']['total_amount_cents'] + + except Exception as e: + logger.error(f"Error analyzing users CSV: {str(e)}") + result['users']['error'] = str(e) + raise + + return result + + # ============================================================================ # CSV Analysis and Preview Generation # ============================================================================ @@ -344,8 +1066,6 @@ def analyze_csv(file_path: str, existing_emails: Optional[set] = None) -> Dict: } } """ - import pandas as pd - # Read CSV with pandas df = pd.read_csv(file_path) @@ -521,11 +1241,4 @@ def format_preview_for_display(preview_data: List[Dict], page: int = 1, page_siz # Module Initialization # ============================================================================ -# Import pandas for CSV processing -try: - import pandas as pd -except ImportError: - logger.error("pandas library not found. Please install: pip install pandas") - raise - logger.info("WordPress parser module loaded successfully")