# views/import_export.py # Unified Import/Export Workflow for all Stiftung content types import csv import io import json from datetime import datetime from decimal import Decimal, InvalidOperation from django.contrib import messages from django.contrib.auth.decorators import login_required from django.http import HttpResponse from django.shortcuts import redirect, render from django.utils import timezone from stiftung.models import ( CSVImport, Destinataer, Foerderung, Land, LandAbrechnung, LandVerpachtung, Paechter, Person, Rentmeister, StiftungsKonto, Verwaltungskosten, ) # --------------------------------------------------------------------------- # Export field definitions for each entity type # Each entry: (csv_header, model_field_or_lambda) # --------------------------------------------------------------------------- def _date_fmt(val): """Format date for CSV export.""" if val is None: return "" return val.strftime("%d.%m.%Y") def _datetime_fmt(val): if val is None: return "" return val.strftime("%d.%m.%Y %H:%M") def _decimal_fmt(val): if val is None: return "" return f"{val:.2f}" def _bool_fmt(val): if val is None: return "" return "ja" if val else "nein" EXPORT_DEFINITIONS = { "destinataere": { "model": Destinataer, "label": "Destinatäre", "queryset": lambda: Destinataer.objects.all().order_by("nachname", "vorname"), "fields": [ ("Vorname", lambda o: o.vorname), ("Nachname", lambda o: o.nachname), ("Geburtsdatum", lambda o: _date_fmt(o.geburtsdatum)), ("E-Mail", lambda o: o.email or ""), ("Telefon", lambda o: o.telefon or ""), ("IBAN", lambda o: o.iban or ""), ("Straße", lambda o: o.strasse or ""), ("PLZ", lambda o: o.plz or ""), ("Ort", lambda o: o.ort or ""), ("Familienzweig", lambda o: o.familienzweig or ""), ("Berufsgruppe", lambda o: o.berufsgruppe or ""), ("Ausbildungsstand", lambda o: o.ausbildungsstand or ""), ("Institution", lambda o: o.institution or ""), ("Projektbeschreibung", lambda o: o.projekt_beschreibung or ""), ("Jährliches_Einkommen", lambda o: _decimal_fmt(o.jaehrliches_einkommen)), ("Finanzielle_Notlage", lambda o: _bool_fmt(o.finanzielle_notlage)), ("Ist_Abkömmling", lambda o: _bool_fmt(o.ist_abkoemmling)), ("Haushaltsgroesse", lambda o: str(o.haushaltsgroesse) if o.haushaltsgroesse else ""), ("Monatliche_Bezuege", lambda o: _decimal_fmt(o.monatliche_bezuege)), ("Vermoegen", lambda o: _decimal_fmt(o.vermoegen)), ("Unterstuetzung_Bestaetigt", lambda o: _bool_fmt(o.unterstuetzung_bestaetigt)), ("Vierteljaehrlicher_Betrag", lambda o: _decimal_fmt(o.vierteljaehrlicher_betrag)), ("Studiennachweis_Erforderlich", lambda o: _bool_fmt(o.studiennachweis_erforderlich)), ("Letzter_Studiennachweis", lambda o: _date_fmt(o.letzter_studiennachweis)), ("Notizen", lambda o: o.notizen or ""), ("Aktiv", lambda o: _bool_fmt(o.aktiv)), ], }, "paechter": { "model": Paechter, "label": "Pächter", "queryset": lambda: Paechter.objects.all().order_by("nachname", "vorname"), "fields": [ ("Vorname", lambda o: o.vorname), ("Nachname", lambda o: o.nachname), ("Geburtsdatum", lambda o: _date_fmt(o.geburtsdatum)), ("E-Mail", lambda o: o.email or ""), ("Telefon", lambda o: o.telefon or ""), ("IBAN", lambda o: o.iban or ""), ("Straße", lambda o: o.strasse or ""), ("PLZ", lambda o: o.plz or ""), ("Ort", lambda o: o.ort or ""), ("Personentyp", lambda o: o.personentyp or ""), ("Pachtnummer", lambda o: o.pachtnummer or ""), ("Pachtbeginn_Erste", lambda o: _date_fmt(o.pachtbeginn_erste)), ("Pachtende_Letzte", lambda o: _date_fmt(o.pachtende_letzte)), ("Pachtzins_Aktuell", lambda o: _decimal_fmt(o.pachtzins_aktuell)), ("Landwirtschaftliche_Ausbildung", lambda o: _bool_fmt(o.landwirtschaftliche_ausbildung)), ("Berufserfahrung_Jahre", lambda o: str(o.berufserfahrung_jahre) if o.berufserfahrung_jahre else ""), ("Spezialisierung", lambda o: o.spezialisierung or ""), ("Notizen", lambda o: o.notizen or ""), ("Aktiv", lambda o: _bool_fmt(o.aktiv)), ], }, "laendereien": { "model": Land, "label": "Ländereien", "queryset": lambda: Land.objects.select_related("aktueller_paechter").all().order_by("lfd_nr"), "fields": [ ("Lfd_Nr", lambda o: o.lfd_nr or ""), ("EW_Nummer", lambda o: o.ew_nummer or ""), ("Grundbuchblatt", lambda o: o.grundbuchblatt or ""), ("ALKIS_Kennzeichen", lambda o: o.alkis_kennzeichen or ""), ("Amtsgericht", lambda o: o.amtsgericht or ""), ("Gemeinde", lambda o: o.gemeinde or ""), ("Gemarkung", lambda o: o.gemarkung or ""), ("Flur", lambda o: o.flur or ""), ("Flurstück", lambda o: o.flurstueck or ""), ("Adresse", lambda o: o.adresse or ""), ("Größe_qm", lambda o: _decimal_fmt(o.groesse_qm)), ("Grünland_qm", lambda o: _decimal_fmt(o.gruenland_qm)), ("Acker_qm", lambda o: _decimal_fmt(o.acker_qm)), ("Wald_qm", lambda o: _decimal_fmt(o.wald_qm)), ("Sonstiges_qm", lambda o: _decimal_fmt(o.sonstiges_qm)), ("Verpachtete_Gesamtfläche_qm", lambda o: _decimal_fmt(o.verpachtete_gesamtflaeche)), ("Verp_Fläche_aktuell_qm", lambda o: _decimal_fmt(o.verp_flaeche_aktuell)), ("Pächter_Name", lambda o: o.paechter_name or ""), ("Pachtbeginn", lambda o: _date_fmt(o.pachtbeginn)), ("Pachtende", lambda o: _date_fmt(o.pachtende)), ("Zahlungsweise", lambda o: o.zahlungsweise or ""), ("Pachtzins_pro_ha", lambda o: _decimal_fmt(o.pachtzins_pro_ha)), ("Pachtzins_pauschal", lambda o: _decimal_fmt(o.pachtzins_pauschal)), ("USt_Option", lambda o: _bool_fmt(o.ust_option)), ("USt_Satz", lambda o: _decimal_fmt(o.ust_satz)), ("Aktiv", lambda o: _bool_fmt(o.aktiv)), ("Notizen", lambda o: o.notizen or ""), ], }, "verpachtungen": { "model": LandVerpachtung, "label": "Verpachtungen", "queryset": lambda: LandVerpachtung.objects.select_related("land", "paechter").all().order_by("vertragsnummer"), "fields": [ ("Vertragsnummer", lambda o: o.vertragsnummer or ""), ("Land_Lfd_Nr", lambda o: o.land.lfd_nr if o.land else ""), ("Land_Gemeinde", lambda o: o.land.gemeinde if o.land else ""), ("Land_Gemarkung", lambda o: o.land.gemarkung if o.land else ""), ("Pächter_Name", lambda o: f"{o.paechter.vorname} {o.paechter.nachname}" if o.paechter else ""), ("Pachtbeginn", lambda o: _date_fmt(o.pachtbeginn)), ("Pachtende", lambda o: _date_fmt(o.pachtende)), ("Verlängerung_Klausel", lambda o: o.verlaengerung_klausel or ""), ("Verpachtete_Fläche_qm", lambda o: _decimal_fmt(o.verpachtete_flaeche)), ("Pachtzins_pauschal", lambda o: _decimal_fmt(o.pachtzins_pauschal)), ("Pachtzins_pro_ha", lambda o: _decimal_fmt(o.pachtzins_pro_ha)), ("Zahlungsweise", lambda o: o.zahlungsweise or ""), ("USt_Option", lambda o: _bool_fmt(o.ust_option)), ("USt_Satz", lambda o: _decimal_fmt(o.ust_satz)), ("Status", lambda o: o.status or ""), ("Bemerkungen", lambda o: o.bemerkungen or ""), ], }, "foerderungen": { "model": Foerderung, "label": "Förderungen", "queryset": lambda: Foerderung.objects.select_related("destinataer").all().order_by("-jahr"), "fields": [ ("Destinatär_Vorname", lambda o: o.destinataer.vorname if o.destinataer else ""), ("Destinatär_Nachname", lambda o: o.destinataer.nachname if o.destinataer else ""), ("Jahr", lambda o: str(o.jahr)), ("Betrag", lambda o: _decimal_fmt(o.betrag)), ("Kategorie", lambda o: o.get_kategorie_display() if o.kategorie else ""), ("Status", lambda o: o.get_status_display() if o.status else ""), ("Antragsdatum", lambda o: _date_fmt(o.antragsdatum)), ("Entscheidungsdatum", lambda o: _date_fmt(o.entscheidungsdatum)), ("Bemerkungen", lambda o: o.bemerkungen or ""), ], }, "konten": { "model": StiftungsKonto, "label": "Stiftungskonten", "queryset": lambda: StiftungsKonto.objects.all().order_by("kontoname"), "fields": [ ("Kontoname", lambda o: o.kontoname), ("Bank", lambda o: o.bank_name or ""), ("IBAN", lambda o: o.iban or ""), ("BIC", lambda o: o.bic or ""), ("Konto_Typ", lambda o: o.get_konto_typ_display() if o.konto_typ else ""), ("Saldo", lambda o: _decimal_fmt(o.saldo)), ("Saldo_Datum", lambda o: _date_fmt(o.saldo_datum)), ("Zinssatz", lambda o: _decimal_fmt(o.zinssatz)), ("Laufzeit_Bis", lambda o: _date_fmt(o.laufzeit_bis)), ("Aktiv", lambda o: _bool_fmt(o.aktiv)), ("Notizen", lambda o: o.notizen or ""), ], }, "verwaltungskosten": { "model": Verwaltungskosten, "label": "Verwaltungskosten", "queryset": lambda: Verwaltungskosten.objects.select_related("rentmeister").all().order_by("-datum"), "fields": [ ("Bezeichnung", lambda o: o.bezeichnung), ("Kategorie", lambda o: o.get_kategorie_display() if o.kategorie else ""), ("Betrag", lambda o: _decimal_fmt(o.betrag)), ("Datum", lambda o: _date_fmt(o.datum)), ("Lieferant", lambda o: o.lieferant_firma or ""), ("Rechnungsnummer", lambda o: o.rechnungsnummer or ""), ("Status", lambda o: o.get_status_display() if o.status else ""), ("Rentmeister", lambda o: f"{o.rentmeister.vorname} {o.rentmeister.nachname}" if o.rentmeister else ""), ("Beschreibung", lambda o: o.beschreibung or ""), ("Notizen", lambda o: o.notizen or ""), ], }, "rentmeister": { "model": Rentmeister, "label": "Rentmeister", "queryset": lambda: Rentmeister.objects.all().order_by("nachname", "vorname"), "fields": [ ("Anrede", lambda o: o.get_anrede_display() if o.anrede else ""), ("Vorname", lambda o: o.vorname), ("Nachname", lambda o: o.nachname), ("Titel", lambda o: o.titel or ""), ("E-Mail", lambda o: o.email or ""), ("Telefon", lambda o: o.telefon or ""), ("Mobil", lambda o: o.mobil or ""), ("Straße", lambda o: o.strasse or ""), ("PLZ", lambda o: o.plz or ""), ("Ort", lambda o: o.ort or ""), ("IBAN", lambda o: o.iban or ""), ("BIC", lambda o: o.bic or ""), ("Bank", lambda o: o.bank_name or ""), ("Seit", lambda o: _date_fmt(o.seit_datum)), ("Bis", lambda o: _date_fmt(o.bis_datum)), ("Monatliche_Vergütung", lambda o: _decimal_fmt(o.monatliche_verguetung)), ("Km_Pauschale", lambda o: _decimal_fmt(o.km_pauschale)), ("Aktiv", lambda o: _bool_fmt(o.aktiv)), ("Notizen", lambda o: o.notizen or ""), ], }, } # --------------------------------------------------------------------------- # Import field definitions for field mapping # Each: (display_label, model_field, field_type, required) # --------------------------------------------------------------------------- IMPORT_FIELD_DEFINITIONS = { "destinataere": { "model": Destinataer, "label": "Destinatäre", "unique_fields": ["vorname", "nachname"], "fields": [ ("Vorname", "vorname", "text", True), ("Nachname", "nachname", "text", True), ("Geburtsdatum", "geburtsdatum", "date", False), ("E-Mail", "email", "text", False), ("Telefon", "telefon", "text", False), ("IBAN", "iban", "text", False), ("Straße", "strasse", "text", False), ("PLZ", "plz", "text", False), ("Ort", "ort", "text", False), ("Familienzweig", "familienzweig", "text", False), ("Berufsgruppe", "berufsgruppe", "text", False), ("Ausbildungsstand", "ausbildungsstand", "text", False), ("Institution", "institution", "text", False), ("Projektbeschreibung", "projekt_beschreibung", "text", False), ("Jährliches Einkommen", "jaehrliches_einkommen", "decimal", False), ("Finanzielle Notlage", "finanzielle_notlage", "bool", False), ("Ist Abkömmling", "ist_abkoemmling", "bool", False), ("Haushaltsgröße", "haushaltsgroesse", "int", False), ("Monatliche Bezüge", "monatliche_bezuege", "decimal", False), ("Vermögen", "vermoegen", "decimal", False), ("Unterstützung bestätigt", "unterstuetzung_bestaetigt", "bool", False), ("Vierteljährlicher Betrag", "vierteljaehrlicher_betrag", "decimal", False), ("Studiennachweis erforderlich", "studiennachweis_erforderlich", "bool", False), ("Letzter Studiennachweis", "letzter_studiennachweis", "date", False), ("Notizen", "notizen", "text", False), ("Aktiv", "aktiv", "bool", False), ], }, "paechter": { "model": Paechter, "label": "Pächter", "unique_fields": ["vorname", "nachname"], "fields": [ ("Vorname", "vorname", "text", True), ("Nachname", "nachname", "text", True), ("Geburtsdatum", "geburtsdatum", "date", False), ("E-Mail", "email", "text", False), ("Telefon", "telefon", "text", False), ("IBAN", "iban", "text", False), ("Straße", "strasse", "text", False), ("PLZ", "plz", "text", False), ("Ort", "ort", "text", False), ("Personentyp", "personentyp", "text", False), ("Pachtnummer", "pachtnummer", "text", False), ("Pachtbeginn Erste", "pachtbeginn_erste", "date", False), ("Pachtende Letzte", "pachtende_letzte", "date", False), ("Pachtzins Aktuell", "pachtzins_aktuell", "decimal", False), ("Landw. Ausbildung", "landwirtschaftliche_ausbildung", "bool", False), ("Berufserfahrung Jahre", "berufserfahrung_jahre", "int", False), ("Spezialisierung", "spezialisierung", "text", False), ("Notizen", "notizen", "text", False), ("Aktiv", "aktiv", "bool", False), ], }, "laendereien": { "model": Land, "label": "Ländereien", "unique_fields": ["lfd_nr"], "unique_fields_alt": ["gemeinde", "gemarkung", "flur", "flurstueck"], "fields": [ ("Lfd Nr", "lfd_nr", "text", False), ("EW Nummer", "ew_nummer", "text", False), ("Grundbuchblatt", "grundbuchblatt", "text", False), ("ALKIS Kennzeichen", "alkis_kennzeichen", "text", False), ("Amtsgericht", "amtsgericht", "text", False), ("Gemeinde", "gemeinde", "text", False), ("Gemarkung", "gemarkung", "text", False), ("Flur", "flur", "text", False), ("Flurstück", "flurstueck", "text", False), ("Adresse", "adresse", "text", False), ("Größe qm", "groesse_qm", "decimal", False), ("Grünland qm", "gruenland_qm", "decimal", False), ("Acker qm", "acker_qm", "decimal", False), ("Wald qm", "wald_qm", "decimal", False), ("Sonstiges qm", "sonstiges_qm", "decimal", False), ("Verpachtete Gesamtfläche", "verpachtete_gesamtflaeche", "decimal", False), ("Verp Fläche aktuell", "verp_flaeche_aktuell", "decimal", False), ("Pächter Name", "paechter_name", "text", False), ("Pächter Anschrift", "paechter_anschrift", "text", False), ("Pachtbeginn", "pachtbeginn", "date", False), ("Pachtende", "pachtende", "date", False), ("Verlängerung Klausel", "verlaengerung_klausel", "text", False), ("Zahlungsweise", "zahlungsweise", "text", False), ("Pachtzins pro ha", "pachtzins_pro_ha", "decimal", False), ("Pachtzins pauschal", "pachtzins_pauschal", "decimal", False), ("USt Option", "ust_option", "bool", False), ("USt Satz", "ust_satz", "decimal", False), ("Grundsteuer Umlage", "grundsteuer_umlage", "bool", False), ("Versicherungen Umlage", "versicherungen_umlage", "bool", False), ("Verbandsbeiträge Umlage", "verbandsbeitraege_umlage", "bool", False), ("Jagdpacht Anteil Umlage", "jagdpacht_anteil_umlage", "bool", False), ("Anteil Grundsteuer", "anteil_grundsteuer", "decimal", False), ("Anteil LWK", "anteil_lwk", "decimal", False), ("Aktiv", "aktiv", "bool", False), ("Notizen", "notizen", "text", False), ], }, "foerderungen": { "model": Foerderung, "label": "Förderungen", "unique_fields": [], "fields": [ ("Destinatär Vorname", "_destinataer_vorname", "text", True), ("Destinatär Nachname", "_destinataer_nachname", "text", True), ("Jahr", "jahr", "int", True), ("Betrag", "betrag", "decimal", True), ("Kategorie", "kategorie", "text", False), ("Status", "status", "text", False), ("Antragsdatum", "antragsdatum", "date", False), ("Entscheidungsdatum", "entscheidungsdatum", "date", False), ("Bemerkungen", "bemerkungen", "text", False), ], }, "konten": { "model": StiftungsKonto, "label": "Stiftungskonten", "unique_fields": ["iban"], "fields": [ ("Kontoname", "kontoname", "text", True), ("Bank", "bank_name", "text", False), ("IBAN", "iban", "text", False), ("BIC", "bic", "text", False), ("Konto Typ", "konto_typ", "text", False), ("Saldo", "saldo", "decimal", False), ("Zinssatz", "zinssatz", "decimal", False), ("Aktiv", "aktiv", "bool", False), ("Notizen", "notizen", "text", False), ], }, "verwaltungskosten": { "model": Verwaltungskosten, "label": "Verwaltungskosten", "unique_fields": [], "fields": [ ("Bezeichnung", "bezeichnung", "text", True), ("Kategorie", "kategorie", "text", False), ("Betrag", "betrag", "decimal", True), ("Datum", "datum", "date", True), ("Lieferant", "lieferant_firma", "text", False), ("Rechnungsnummer", "rechnungsnummer", "text", False), ("Status", "status", "text", False), ("Beschreibung", "beschreibung", "text", False), ("Notizen", "notizen", "text", False), ], }, "rentmeister": { "model": Rentmeister, "label": "Rentmeister", "unique_fields": ["vorname", "nachname"], "fields": [ ("Anrede", "anrede", "text", False), ("Vorname", "vorname", "text", True), ("Nachname", "nachname", "text", True), ("Titel", "titel", "text", False), ("E-Mail", "email", "text", False), ("Telefon", "telefon", "text", False), ("Mobil", "mobil", "text", False), ("Straße", "strasse", "text", False), ("PLZ", "plz", "text", False), ("Ort", "ort", "text", False), ("IBAN", "iban", "text", False), ("BIC", "bic", "text", False), ("Bank", "bank_name", "text", False), ("Seit", "seit_datum", "date", False), ("Bis", "bis_datum", "date", False), ("Monatliche Vergütung", "monatliche_verguetung", "decimal", False), ("Km Pauschale", "km_pauschale", "decimal", False), ("Aktiv", "aktiv", "bool", False), ("Notizen", "notizen", "text", False), ], }, } # --------------------------------------------------------------------------- # Value parsing helpers # --------------------------------------------------------------------------- def _parse_bool(value): if not value: return None v = str(value).strip().lower() if v in ("true", "ja", "yes", "1", "wahr", "x"): return True if v in ("false", "nein", "no", "0", "falsch", ""): return False return None def _parse_date(value): if not value or not str(value).strip(): return None v = str(value).strip() for fmt in ("%d.%m.%Y", "%Y-%m-%d", "%d/%m/%Y"): try: return datetime.strptime(v, fmt).date() except ValueError: continue return None def _parse_decimal(value): if not value or not str(value).strip(): return None v = str(value).strip().replace(",", ".") try: return Decimal(v) except (InvalidOperation, ValueError): return None def _parse_int(value): if not value or not str(value).strip(): return None try: return int(str(value).strip().replace(",", "").replace(".", "")) except ValueError: return None def _parse_value(raw, field_type): """Parse a raw CSV string into the appropriate Python type.""" if field_type == "date": return _parse_date(raw) elif field_type == "decimal": return _parse_decimal(raw) elif field_type == "int": return _parse_int(raw) elif field_type == "bool": return _parse_bool(raw) else: return str(raw).strip() if raw else None # --------------------------------------------------------------------------- # Views # --------------------------------------------------------------------------- @login_required def import_export_hub(request): """Unified import/export hub page.""" # Get recent imports for display recent_imports = CSVImport.objects.all().order_by("-started_at")[:10] # Count records per entity type entity_counts = {} for key, defn in EXPORT_DEFINITIONS.items(): try: entity_counts[key] = defn["model"].objects.count() except Exception: entity_counts[key] = 0 export_types = [ {"key": k, "label": v["label"], "count": entity_counts.get(k, 0)} for k, v in EXPORT_DEFINITIONS.items() ] import_types = [ {"key": k, "label": v["label"]} for k, v in IMPORT_FIELD_DEFINITIONS.items() ] context = { "export_types": export_types, "import_types": import_types, "recent_imports": recent_imports, } return render(request, "stiftung/import_export_hub.html", context) @login_required def csv_export(request): """Export any entity type as CSV.""" export_type = request.GET.get("type") if export_type not in EXPORT_DEFINITIONS: messages.error(request, "Unbekannter Export-Typ.") return redirect("stiftung:import_export_hub") defn = EXPORT_DEFINITIONS[export_type] queryset = defn["queryset"]() response = HttpResponse(content_type="text/csv; charset=utf-8") response["Content-Disposition"] = f'attachment; filename="{export_type}_{timezone.now().strftime("%Y%m%d_%H%M")}.csv"' # BOM for Excel compatibility response.write("\ufeff") writer = csv.writer(response, delimiter=";") headers = [f[0] for f in defn["fields"]] writer.writerow(headers) for obj in queryset: row = [] for _header, extractor in defn["fields"]: try: row.append(extractor(obj)) except Exception: row.append("") writer.writerow(row) return response @login_required def csv_import_upload(request): """Step 1: Upload CSV file and show field mapping UI.""" if request.method != "POST": return redirect("stiftung:import_export_hub") import_type = request.POST.get("import_type") csv_file = request.FILES.get("csv_file") if not csv_file or not import_type: messages.error(request, "Bitte wählen Sie einen Import-Typ und eine CSV-Datei aus.") return redirect("stiftung:import_export_hub") if not csv_file.name.lower().endswith(".csv"): messages.error(request, "Bitte wählen Sie eine gültige CSV-Datei aus.") return redirect("stiftung:import_export_hub") if import_type not in IMPORT_FIELD_DEFINITIONS: messages.error(request, "Unbekannter Import-Typ.") return redirect("stiftung:import_export_hub") defn = IMPORT_FIELD_DEFINITIONS[import_type] try: raw_bytes = csv_file.read() # Try UTF-8 first, fallback to latin-1 try: decoded = raw_bytes.decode("utf-8-sig") except UnicodeDecodeError: decoded = raw_bytes.decode("latin-1") # Detect delimiter first_line = decoded.split("\n")[0] delimiter = ";" if ";" in first_line else "," reader = csv.reader(io.StringIO(decoded), delimiter=delimiter) csv_headers = next(reader) csv_headers = [h.strip() for h in csv_headers] # Read preview rows (up to 5) preview_rows = [] for i, row in enumerate(reader): if i >= 5: break preview_rows.append(row) # Count remaining rows remaining = sum(1 for _ in reader) total_rows = len(preview_rows) + remaining # Auto-match CSV headers to model fields # Build mapping suggestions based on fuzzy matching model_fields = defn["fields"] # list of (label, field_name, type, required) auto_mapping = {} for csv_idx, csv_header in enumerate(csv_headers): csv_h_lower = csv_header.lower().replace("_", " ").replace("-", " ").strip() best_match = "" best_score = 0 for label, field_name, _ftype, _req in model_fields: label_lower = label.lower().replace("_", " ").replace("-", " ").strip() field_lower = field_name.lower().replace("_", " ").strip() # Exact match if csv_h_lower == label_lower or csv_h_lower == field_lower: best_match = field_name best_score = 100 break # Partial match if csv_h_lower in label_lower or label_lower in csv_h_lower: score = 80 if score > best_score: best_match = field_name best_score = score elif csv_h_lower in field_lower or field_lower in csv_h_lower: score = 70 if score > best_score: best_match = field_name best_score = score if best_score >= 70: auto_mapping[str(csv_idx)] = best_match # Build header_previews: list of dicts with header + first-row preview header_previews = [] first_row = preview_rows[0] if preview_rows else [] for idx, header in enumerate(csv_headers): preview_val = first_row[idx] if idx < len(first_row) else "" header_previews.append({"header": header, "preview": preview_val}) # Store CSV data in session for step 2 request.session["csv_import_data"] = decoded request.session["csv_import_delimiter"] = delimiter request.session["csv_import_type"] = import_type request.session["csv_import_filename"] = csv_file.name request.session["csv_import_filesize"] = len(raw_bytes) context = { "import_type": import_type, "import_label": defn["label"], "header_previews": header_previews, "model_fields": model_fields, "preview_rows": preview_rows, "total_rows": total_rows, "filename": csv_file.name, "auto_mapping_json": json.dumps(auto_mapping), } return render(request, "stiftung/csv_import_mapping.html", context) except Exception as e: messages.error(request, f"Fehler beim Lesen der CSV-Datei: {str(e)}") return redirect("stiftung:import_export_hub") @login_required def csv_import_execute(request): """Step 2: Execute the import with user-defined field mapping.""" if request.method != "POST": return redirect("stiftung:import_export_hub") import_type = request.session.get("csv_import_type") csv_data = request.session.get("csv_import_data") delimiter = request.session.get("csv_import_delimiter", ",") filename = request.session.get("csv_import_filename", "unknown.csv") filesize = request.session.get("csv_import_filesize", 0) if not import_type or not csv_data: messages.error(request, "Keine Import-Daten gefunden. Bitte starten Sie den Import erneut.") return redirect("stiftung:import_export_hub") if import_type not in IMPORT_FIELD_DEFINITIONS: messages.error(request, "Unbekannter Import-Typ.") return redirect("stiftung:import_export_hub") defn = IMPORT_FIELD_DEFINITIONS[import_type] # Import mode: merge (update existing), skip (skip existing), create (always new) import_mode = request.POST.get("import_mode", "merge") if import_mode not in ("merge", "skip", "create"): import_mode = "merge" # Parse field mapping from POST data # Format: mapping_0=field_name, mapping_1=field_name, ... reader = csv.reader(io.StringIO(csv_data), delimiter=delimiter) csv_headers = next(reader) csv_headers = [h.strip() for h in csv_headers] field_mapping = {} # csv_index -> (model_field, field_type) field_types = {f[1]: f[2] for f in defn["fields"]} for i in range(len(csv_headers)): mapped_field = request.POST.get(f"mapping_{i}", "") if mapped_field and mapped_field != "__skip__": ftype = field_types.get(mapped_field, "text") field_mapping[i] = (mapped_field, ftype) if not field_mapping: messages.error(request, "Keine Felder zugeordnet. Bitte ordnen Sie mindestens ein Feld zu.") return redirect("stiftung:import_export_hub") # Check required fields - warn but don't block (per-row validation will handle it) required_fields = {f[1] for f in defn["fields"] if f[3]} mapped_fields = {v[0] for v in field_mapping.values()} missing_required = required_fields - mapped_fields missing_required = {f for f in missing_required if not f.startswith("_")} if missing_required: field_labels = {f[1]: f[0] for f in defn["fields"]} missing_labels = [field_labels.get(f, f) for f in missing_required] messages.warning( request, f"Hinweis: Pflichtfelder nicht zugeordnet: {', '.join(missing_labels)}. " f"Zeilen ohne diese Daten werden übersprungen." ) # Create import record csv_import = CSVImport.objects.create( import_type=import_type, filename=filename, file_size=filesize, created_by=request.user.username if request.user.is_authenticated else "Unknown", status="processing", ) # Process the import model = defn["model"] unique_fields = defn["unique_fields"] unique_fields_alt = defn.get("unique_fields_alt", []) total_rows = 0 imported_rows = 0 failed_rows = 0 skipped_rows = 0 error_log = [] reader = csv.reader(io.StringIO(csv_data), delimiter=delimiter) next(reader) # Skip header for row_num, row in enumerate(reader, start=2): total_rows += 1 try: # Build data dict from mapping data = {} for csv_idx, (model_field, field_type) in field_mapping.items(): if csv_idx < len(row): raw_value = row[csv_idx] parsed = _parse_value(raw_value, field_type) data[model_field] = parsed # Special handling for Förderungen (link to Destinatär) if import_type == "foerderungen": vorname = data.pop("_destinataer_vorname", None) nachname = data.pop("_destinataer_nachname", None) if vorname and nachname: dest = Destinataer.objects.filter( vorname__iexact=vorname, nachname__iexact=nachname ).first() if dest: data["destinataer"] = dest else: error_log.append( f"Zeile {row_num}: Destinatär '{vorname} {nachname}' nicht gefunden" ) failed_rows += 1 continue else: error_log.append(f"Zeile {row_num}: Destinatär Vor-/Nachname erforderlich") failed_rows += 1 continue # Validate required fields required_missing = [] for label, field_name, _ftype, required in defn["fields"]: if required and not field_name.startswith("_"): val = data.get(field_name) if val is None or (isinstance(val, str) and not val.strip()): required_missing.append(label) if required_missing: error_log.append( f"Zeile {row_num}: Pflichtfelder leer: {', '.join(required_missing)}" ) failed_rows += 1 continue # Clean None values from data - don't set fields that weren't mapped clean_data = {k: v for k, v in data.items() if v is not None and not k.startswith("_")} # For fields not in clean_data, check if the DB column requires # a value (NOT NULL without a model default). If so, provide a # sensible zero-value so the INSERT doesn't fail. for fname, ftype in ((f[1], f[2]) for f in defn["fields"]): if fname.startswith("_") or fname in clean_data: continue try: mf = model._meta.get_field(fname) if not mf.null and not mf.has_default(): if ftype == "decimal": clean_data[fname] = Decimal("0") elif ftype == "int": clean_data[fname] = 0 elif ftype == "bool": clean_data[fname] = False elif ftype == "text": # For unique text fields, generate a value # instead of empty string to avoid unique violations if mf.unique: import uuid as _uuid clean_data[fname] = f"AUTO-{_uuid.uuid4().hex[:8]}" else: clean_data[fname] = "" except Exception: pass # Try to find existing record using user-mapped data (not auto-generated defaults) existing = None if import_mode != "create": # Use original user data (before defaults) for dedup lookup for uf_set in ([unique_fields] if unique_fields else []) + ([unique_fields_alt] if unique_fields_alt else []): if existing: break lookup = {} for uf in uf_set: val = data.get(uf) if val and (not isinstance(val, str) or val.strip()): lookup[f"{uf}__iexact"] = val if isinstance(val, str) else val else: lookup = None break if lookup: existing = model.objects.filter(**lookup).first() if existing: if import_mode == "skip": skipped_rows += 1 continue else: # Merge mode: update existing record with mapped values for field, value in clean_data.items(): setattr(existing, field, value) existing.save() else: model.objects.create(**clean_data) imported_rows += 1 except Exception as e: error_log.append(f"Zeile {row_num}: {str(e)}") failed_rows += 1 # Determine status if failed_rows == 0 and (imported_rows > 0 or skipped_rows > 0): status = "completed" elif imported_rows > 0 or skipped_rows > 0: status = "partial" elif total_rows == 0: status = "completed" else: status = "failed" # Build skip info for error log if skipped_rows > 0: error_log.insert(0, f"Übersprungen: {skipped_rows} bereits vorhandene Einträge") # Update import record csv_import.total_rows = total_rows csv_import.imported_rows = imported_rows csv_import.failed_rows = failed_rows csv_import.error_log = "\n".join(error_log) if error_log else None csv_import.status = status csv_import.completed_at = timezone.now() csv_import.save() # Clean session for key in ["csv_import_data", "csv_import_delimiter", "csv_import_type", "csv_import_filename", "csv_import_filesize"]: request.session.pop(key, None) skip_info = f", {skipped_rows} übersprungen" if skipped_rows > 0 else "" if status == "completed": messages.success( request, f"Import erfolgreich! {imported_rows} Datensätze importiert{skip_info}.", ) elif status == "partial": messages.warning( request, f"Import teilweise erfolgreich. {imported_rows} importiert, {failed_rows} fehlgeschlagen{skip_info}.", ) else: messages.error( request, f"Import fehlgeschlagen. {failed_rows} Zeilen konnten nicht importiert werden{skip_info}.", ) return redirect("stiftung:import_export_hub")