From a8770abd3a1cfd0749e411ca8b2fea15e45b82ea Mon Sep 17 00:00:00 2001 From: SimolZimol <70102430+SimolZimol@users.noreply.github.com> Date: Sun, 24 Aug 2025 01:21:39 +0200 Subject: [PATCH] modified: app.py modified: bot.py --- app.py | 161 ++++++++++++++++++++----------- bot.py | 2 +- database_optimization_plan.md | 177 ++++++++++++++++++++++++++++++++++ 3 files changed, 284 insertions(+), 56 deletions(-) create mode 100644 database_optimization_plan.md diff --git a/app.py b/app.py index 6738a60..ca69e37 100644 --- a/app.py +++ b/app.py @@ -8,6 +8,7 @@ import os import subprocess import psutil import mysql.connector +import mysql.connector.pooling from datetime import datetime, timedelta from flask_session import Session import logging @@ -85,15 +86,68 @@ def stop_bot(): else: print("Bot läuft nicht.") +# Database Connection Pool für bessere Verbindungsverwaltung +app_pool = mysql.connector.pooling.MySQLConnectionPool( + pool_name="app_pool", + pool_size=15, # Reduziert von 20 auf 15 (Bot: 30, App: 15 = max 45 statt 50+) + pool_reset_session=True, + host=DB_HOST, + port=DB_PORT, + user=DB_USER, + password=DB_PASS, + database=DB_NAME, + # Zusätzliche Pool-Optimierungen + use_pure=True, # Verwende Pure Python Connector für bessere Stabilität + connect_timeout=10, # Timeout für Verbindungsaufbau + autocommit=True, # Für bessere Pool-Performance +) + def get_db_connection(): - """Stellt eine Verbindung zur MySQL-Datenbank her.""" - return mysql.connector.connect( - host=DB_HOST, - port=DB_PORT, - user=DB_USER, - password=DB_PASS, - database=DB_NAME - ) + """Stellt eine Verbindung zur MySQL-Datenbank über Connection Pool her.""" + try: + connection = app_pool.get_connection() + return connection + except mysql.connector.PoolError as e: + print(f"Pool error in app.py: {e}") + # Fallback zu direkter Verbindung bei Pool-Problemen + return mysql.connector.connect( + host=DB_HOST, + port=DB_PORT, + user=DB_USER, + password=DB_PASS, + database=DB_NAME + ) + +from contextlib import contextmanager + +@contextmanager +def get_db_cursor(): + """Context Manager für sichere Datenbankverbindungen mit automatischer Bereinigung.""" + connection = None + cursor = None + try: + connection = get_db_connection() + cursor = connection.cursor(dictionary=True) + yield cursor, connection + except Exception as e: + if connection: + connection.rollback() + raise e + finally: + if cursor: + cursor.close() + if connection and connection.is_connected(): + connection.close() + +def get_pool_status(): + """Gibt Pool-Status zurück für Monitoring""" + try: + return { + "pool_size": app_pool.pool_size, + "connections_in_use": len(app_pool._cnx_queue._queue) if hasattr(app_pool, '_cnx_queue') else 'Unknown' + } + except Exception as e: + return {"error": str(e)} def token_updater(token): session['oauth_token'] = token @@ -146,15 +200,14 @@ def is_bot_admin(): user_info = session["discord_user"] user_id = user_info["id"] - connection = get_db_connection() - cursor = connection.cursor(dictionary=True) - cursor.execute("SELECT global_permission FROM bot_data WHERE user_id = %s", (user_id,)) - user_data = cursor.fetchone() - - cursor.close() - connection.close() - - return user_data and user_data["global_permission"] >= 8 + try: + with get_db_cursor() as (cursor, connection): + cursor.execute("SELECT global_permission FROM bot_data WHERE user_id = %s", (user_id,)) + user_data = cursor.fetchone() + return user_data and user_data["global_permission"] >= 8 + except Exception as e: + print(f"Database error in is_bot_admin: {e}") + return False return False def is_server_admin(guild_id): @@ -163,15 +216,14 @@ def is_server_admin(guild_id): user_info = session["discord_user"] user_id = user_info["id"] - connection = get_db_connection() - cursor = connection.cursor(dictionary=True) - cursor.execute("SELECT permission FROM user_data WHERE user_id = %s AND guild_id = %s", (user_id, guild_id)) - user_data = cursor.fetchone() - - cursor.close() - connection.close() - - return user_data and user_data["permission"] >= 8 + try: + with get_db_cursor() as (cursor, connection): + cursor.execute("SELECT permission FROM user_data WHERE user_id = %s AND guild_id = %s", (user_id, guild_id)) + user_data = cursor.fetchone() + return user_data and user_data["permission"] >= 8 + except Exception as e: + print(f"Database error in is_server_admin: {e}") + return False return False @app.route("/") @@ -517,38 +569,37 @@ def user_dashboard(guild_id): def leaderboard(guild_id): """Zeigt das Level Leaderboard für einen bestimmten Server an.""" if "discord_user" in session: - connection = get_db_connection() - cursor = connection.cursor(dictionary=True) + try: + with get_db_cursor() as (cursor, connection): + current_date = datetime.now() + one_month_ago = current_date - timedelta(days=30) - current_date = datetime.now() - one_month_ago = current_date - timedelta(days=30) + # Hole die Leaderboard-Daten + cursor.execute(""" + SELECT nickname, profile_picture, level, xp, join_date + FROM user_data + WHERE guild_id = %s + AND ban = 0 + AND (leave_date IS NULL OR leave_date > %s) + ORDER BY level DESC, xp DESC + """, (guild_id, one_month_ago)) + + leaderboard_data = cursor.fetchall() - # Hole die Leaderboard-Daten - cursor.execute(""" - SELECT nickname, profile_picture, level, xp, join_date - FROM user_data - WHERE guild_id = %s - AND ban = 0 - AND (leave_date IS NULL OR leave_date > %s) - ORDER BY level DESC, xp DESC - """, (guild_id, one_month_ago)) - - leaderboard_data = cursor.fetchall() + # Hole den Server-Namen aus der guilds-Tabelle + cursor.execute("SELECT name FROM guilds WHERE guild_id = %s", (guild_id,)) + guild_name_result = cursor.fetchone() + guild_name = guild_name_result["name"] if guild_name_result else f"Server {guild_id}" - # Hole den Server-Namen aus der guilds-Tabelle - cursor.execute("SELECT name FROM guilds WHERE guild_id = %s", (guild_id,)) - guild_name_result = cursor.fetchone() - guild_name = guild_name_result["name"] if guild_name_result else f"Server {guild_id}" - - cursor.close() - connection.close() - - # Übergabe von enumerate und guild_name an das Template - return render_template("leaderboard.html", - leaderboard=leaderboard_data, - guild_id=guild_id, - guild_name=guild_name, - enumerate=enumerate) + # Übergabe von enumerate und guild_name an das Template + return render_template("leaderboard.html", + leaderboard=leaderboard_data, + guild_id=guild_id, + guild_name=guild_name, + enumerate=enumerate) + except Exception as e: + print(f"Database error in leaderboard: {e}") + return "Database connection error", 500 return redirect(url_for("landing_page")) diff --git a/bot.py b/bot.py index ec2565a..fbe4c8e 100644 --- a/bot.py +++ b/bot.py @@ -196,7 +196,7 @@ def retry_query(func, *args, retries=3, delay=5): pool = mysql.connector.pooling.MySQLConnectionPool( pool_name="mypool", - pool_size=30, # Erhöht von 10 auf 30 + pool_size=25, # Reduziert von 30 auf 25 (App: 15, Bot: 25 = 40 total) pool_reset_session=True, autocommit=True, host=DB_HOST, diff --git a/database_optimization_plan.md b/database_optimization_plan.md new file mode 100644 index 0000000..b19350b --- /dev/null +++ b/database_optimization_plan.md @@ -0,0 +1,177 @@ +# Database Connection Management Improvements for Bot +# Diese Datei zeigt die empfohlenen Änderungen für bot.py um das "Too many connections" Problem zu lösen + +## Problem Analyse: +# 1. Bot Pool: 30 Verbindungen +# 2. App direktverbindungen ohne Pool (jetzt mit Pool: 15) +# 3. Neue Warning-Funktionen verwenden viele DB-Verbindungen +# 4. get_user_warnings() wird häufig aufgerufen und öffnet jedes Mal neue Connections +# 5. Context-Archivierung kann große Datenmengen verarbeiten + +## Lösungsansätze implementiert: + +### 1. App.py Connection Pool (✅ Implementiert): +- Connection Pool mit 15 Verbindungen für Flask App +- Context Manager für sichere Verbindungsverwaltung +- Automatische Verbindungsfreigabe +- Fallback für Pool-Probleme + +### 2. Optimierungen für Bot.py (Empfohlen): +# Diese Änderungen sollten in bot.py implementiert werden: + +```python +# Verbesserte get_user_warnings Funktion mit Connection Pooling +async def get_user_warnings(user_id, guild_id, active_only=True): + """Retrieves warning records for a user - OPTIMIZED VERSION""" + connection = None + cursor = None + try: + connection = connect_to_database() # Nutzt bereits den Pool + cursor = connection.cursor() + + # Single query statt multiple calls + select_query = """ + SELECT id, moderator_id, reason, created_at, message_id, message_content, + message_attachments, message_author_id, message_channel_id, context_messages, aktiv + FROM user_warnings + WHERE user_id = %s AND guild_id = %s {} + ORDER BY created_at DESC + """.format("AND aktiv = TRUE" if active_only else "") + + cursor.execute(select_query, (user_id, guild_id)) + results = cursor.fetchall() + + warnings = [] + for row in results: + warnings.append({ + "id": row[0], + "moderator_id": row[1], + "reason": row[2], + "created_at": row[3], + "message_id": row[4], + "message_content": row[5], + "message_attachments": row[6], + "message_author_id": row[7], + "message_channel_id": row[8], + "context_messages": row[9], + "aktiv": row[10] + }) + + return warnings + + except Exception as e: + logger.error(f"Error getting user warnings: {e}") + return [] + finally: + if cursor: + cursor.close() + if connection: + close_database_connection(connection) # Gibt Connection an Pool zurück +``` + +### 3. Connection Caching für häufige Abfragen: +# Implementiere Caching für Warning-Abfragen: + +```python +import asyncio +from functools import lru_cache + +# Cache für häufige Warning-Abfragen (5 Minuten TTL) +warning_cache = {} +cache_ttl = 300 # 5 Minuten + +async def get_user_warnings_cached(user_id, guild_id, active_only=True): + """Cached version of get_user_warnings""" + cache_key = f"{user_id}_{guild_id}_{active_only}" + current_time = asyncio.get_event_loop().time() + + # Check cache + if cache_key in warning_cache: + cached_data, timestamp = warning_cache[cache_key] + if current_time - timestamp < cache_ttl: + return cached_data + + # Fetch fresh data + warnings = await get_user_warnings(user_id, guild_id, active_only) + warning_cache[cache_key] = (warnings, current_time) + + # Clean old cache entries + if len(warning_cache) > 1000: # Limit cache size + old_keys = [k for k, (_, ts) in warning_cache.items() + if current_time - ts > cache_ttl] + for k in old_keys: + del warning_cache[k] + + return warnings +``` + +### 4. Batch Operations für Context Messages: +# Reduziere DB-Aufrufe bei Context-Archivierung: + +```python +async def batch_insert_warnings(warning_data_list): + """Insert multiple warnings in a single transaction""" + if not warning_data_list: + return + + connection = None + cursor = None + try: + connection = connect_to_database() + cursor = connection.cursor() + + insert_query = """ + INSERT INTO user_warnings (user_id, guild_id, moderator_id, reason, created_at, + message_id, message_content, message_attachments, + message_author_id, message_channel_id, context_messages, aktiv) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + """ + + cursor.executemany(insert_query, warning_data_list) + connection.commit() + + except Exception as e: + logger.error(f"Error in batch insert warnings: {e}") + if connection: + connection.rollback() + finally: + if cursor: + cursor.close() + if connection: + close_database_connection(connection) +``` + +### 5. Pool Monitoring: +# Überwache Pool-Status: + +```python +def monitor_connection_pool(): + """Monitor connection pool status""" + try: + pool_size = pool.pool_size + # This is tricky to get exact usage, but we can log pool creation + logger.info(f"Connection pool status - Size: {pool_size}") + return pool_size + except Exception as e: + logger.error(f"Error monitoring pool: {e}") + return 0 +``` + +## Sofortige Maßnahmen: +1. ✅ App.py mit Connection Pool ausgestattet (15 Verbindungen) +2. 🔄 Bot Pool von 30 auf 25 reduzieren (Gesamtlimit: 40 statt 50+) +3. 🔄 Warning-Cache implementieren +4. 🔄 Batch-Operations für große Datensätze + +## Connection Limits: +- MySQL Standard: 151 gleichzeitige Verbindungen +- Bot Pool: 30 → empfohlen 25 +- App Pool: 15 +- Reserve für andere Clients: 111 +- Sicherheitspuffer: sollte ausreichend sein + +Das Problem tritt auf, weil: +1. Neue Warning-Funktionen häufige DB-Zugriffe machen +2. Context-Archivierung große Datenmengen verarbeitet +3. get_user_warnings() wird oft aufgerufen (account, viewwarn Commands) +4. App und Bot konkurrieren um Verbindungen