Files
offline-db/aniscrap_core/settings.py
Barış Keserandgoogle-labs-jules[bot] 5dde0cafbf perf: optimize N+1 query in encoder and fansub wallet updates (#350)
Refactored the `calculate_revenue` task in `billing/tasks.py` to eliminate N+1
database queries when distributing revenue to encoders and fansub group owners.

Changes:
- Replaced individual `Wallet.objects.get_or_create` and `wallet.save()` calls
  inside loops with bulk operations.
- Implemented a 'fetch-create-update' pattern:
  1. Identify all required user IDs.
  2. Bulk-create missing `Wallet` objects.
  3. Fetch all relevant wallets in a single query.
  4. Perform in-memory balance updates.
  5. Use `bulk_update` to persist changes in a single query per pool.
- Cleaned up unused imports (`VideoFile`) and variables.
- Ensured `python-dotenv` is an optional dependency in `manage.py` and
  `aniscrap_core/settings.py` for improved environment resilience.

This optimization reduces the number of database queries from O(N) to O(1)
per distribution pool, significantly improving performance for tasks with
many unique revenue recipients.

Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com>
2026-04-21 15:25:32 +00:00

359 lines
10 KiB
Python

"""
Django settings for AniScrap project.
"""
import os
from pathlib import Path
import logging
from celery.schedules import crontab
try:
from dotenv import load_dotenv
load_dotenv()
except ImportError:
pass
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.getenv('DJANGO_SECRET_KEY')
if not SECRET_KEY:
if os.getenv('DEBUG', 'False') == 'True':
SECRET_KEY = 'django-insecure-aniscrap-dev-key'
else:
from django.core.exceptions import ImproperlyConfigured
raise ImproperlyConfigured("The DJANGO_SECRET_KEY setting must not be empty.")
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = os.getenv('DEBUG', 'False') == 'True'
ALLOWED_HOSTS = os.getenv('ALLOWED_HOSTS', '127.0.0.1,localhost').split(',')
# Security Hardening
SECURE_BROWSER_XSS_FILTER = True
SECURE_CONTENT_TYPE_NOSNIFF = True
X_FRAME_OPTIONS = 'DENY'
SECURE_REFERRER_POLICY = 'strict-origin-when-cross-origin'
# Session Cookie Security
SESSION_COOKIE_SECURE = True
SESSION_COOKIE_HTTPONLY = True
SESSION_COOKIE_SAMESITE = 'Lax'
# CSRF Trusted Origins
CSRF_TRUSTED_ORIGINS = os.getenv('CSRF_TRUSTED_ORIGINS', 'https://localhost,https://127.0.0.1').split(',')
if not DEBUG:
# SECURE_SSL_REDIRECT = True
CSRF_COOKIE_SECURE = True
SECURE_HSTS_SECONDS = 31536000 # 1 year
SECURE_HSTS_INCLUDE_SUBDOMAINS = True
SECURE_HSTS_PRELOAD = True
# Application definition
INSTALLED_APPS = [
"daphne",
"channels",
# Dependencies for Unfold contrib (Kept for compatibility if used elsewhere, but Unfold removed)
"import_export",
"guardian",
"simple_history",
# Third Party
"rest_framework",
"rest_framework_simplejwt",
"django_filters",
"corsheaders",
"django_celery_results",
"drf_spectacular",
"unfold", # Added Unfold Theme
"django.contrib.admin",
"django.contrib.auth",
"django.contrib.contenttypes",
"django.contrib.sessions",
"django.contrib.messages",
"django.contrib.staticfiles",
"django.contrib.sites",
"django.contrib.sitemaps",
# Local Apps
"core",
"content",
"users",
"billing",
"scraper_module",
"apps.watchparty",
]
SITE_ID = 1
AUTH_USER_MODEL = "users.User"
MIDDLEWARE = [
"django.middleware.security.SecurityMiddleware",
"core.middleware.SecurityHeadersMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"corsheaders.middleware.CorsMiddleware",
"django.middleware.common.CommonMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
"csp.middleware.CSPMiddleware",
"aniscrap_core.middleware.security.SecurityHeadersMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
"django.middleware.clickjacking.XFrameOptionsMiddleware",
]
ROOT_URLCONF = "aniscrap_core.urls"
TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",
"DIRS": [BASE_DIR / 'templates'],
"APP_DIRS": True,
"OPTIONS": {
"context_processors": [
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
"django.template.context_processors.media",
"core.context_processors.site_settings", # I'll also add this for Footer
],
},
},
]
WSGI_APPLICATION = "aniscrap_core.wsgi.application"
ASGI_APPLICATION = "aniscrap_core.asgi.application"
# Database
# https://docs.djangoproject.com/en/6.0/ref/settings/#databases
USE_SQLITE = os.getenv('USE_SQLITE', 'False') == 'True'
if USE_SQLITE:
DATABASES = {
"default": {
"ENGINE": "django.db.backends.sqlite3",
"NAME": BASE_DIR / "db.sqlite3",
}
}
else:
DATABASES = {
"default": {
"ENGINE": "django.db.backends.mysql",
"NAME": os.getenv('DB_NAME', 'aniscrap'),
"USER": os.getenv('DB_USER', 'root'),
"PASSWORD": os.getenv('DB_PASSWORD', ''),
"HOST": os.getenv('DB_HOST', '127.0.0.1'),
"PORT": os.getenv('DB_PORT', '3306'),
'OPTIONS': {
'init_command': "SET sql_mode='STRICT_TRANS_TABLES'",
'charset': 'utf8mb4',
},
}
}
# Authentication Backends
AUTHENTICATION_BACKENDS = (
"django.contrib.auth.backends.ModelBackend",
"guardian.backends.ObjectPermissionBackend",
)
# Password Hashing
PASSWORD_HASHERS = [
"django.contrib.auth.hashers.Argon2PasswordHasher",
"django.contrib.auth.hashers.PBKDF2PasswordHasher",
"django.contrib.auth.hashers.PBKDF2SHA1PasswordHasher",
"django.contrib.auth.hashers.BCryptSHA256PasswordHasher",
]
# Password validation
AUTH_PASSWORD_VALIDATORS = [
{
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
},
{
"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
},
{
"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
},
{
"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
},
]
# Internationalization
LANGUAGE_CODE = "en-us"
TIME_ZONE = "UTC"
USE_I18N = True
USE_TZ = True
LANGUAGES = [
('en', 'English'),
('tr', 'Turkish'),
]
LOCALE_PATHS = [BASE_DIR / 'locale']
# Static files (CSS, JavaScript, Images)
STATIC_URL = "static/"
STATIC_ROOT = BASE_DIR / "staticfiles"
STATICFILES_DIRS = [BASE_DIR / "static"]
# Media files
MEDIA_URL = "media/"
MEDIA_ROOT = BASE_DIR / "media"
# Caching - Native Django cache framework configured for Redis Caching Strategy
if os.getenv('USE_SQLITE', 'False') == 'True':
CACHES = {
"default": {
"BACKEND": "django.core.cache.backends.locmem.LocMemCache",
}
}
CHANNEL_LAYERS = {
"default": {
"BACKEND": "channels.layers.InMemoryChannelLayer"
}
}
else:
CACHES = {
"default": {
"BACKEND": "django.core.cache.backends.redis.RedisCache",
"LOCATION": os.getenv('REDIS_URL', 'redis://127.0.0.1:6379/1'),
}
}
CHANNEL_LAYERS = {
"default": {
"BACKEND": "channels_redis.core.RedisChannelLayer",
"CONFIG": {
"hosts": [os.getenv('REDIS_URL', 'redis://127.0.0.1:6379/1')],
},
}
}
# Celery Configuration
if USE_SQLITE:
CELERY_BROKER_URL = 'memory://'
CELERY_TASK_ALWAYS_EAGER = True
else:
CELERY_BROKER_URL = os.getenv('CELERY_BROKER_URL', 'redis://127.0.0.1:6379/0')
CELERY_RESULT_BACKEND = 'django-db'
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TIMEZONE = TIME_ZONE
# Email Configuration
EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
EMAIL_HOST = os.getenv('EMAIL_HOST', 'smtp.gmail.com')
EMAIL_PORT = int(os.getenv('EMAIL_PORT', 587))
EMAIL_USE_TLS = True
EMAIL_HOST_USER = os.getenv('EMAIL_USER')
EMAIL_HOST_PASSWORD = os.getenv('EMAIL_PASSWORD')
DEFAULT_FROM_EMAIL = EMAIL_HOST_USER
# Footer Details
SITE_NAME = "AniScrap"
SITE_URL = os.getenv('SITE_URL', 'http://127.0.0.1:8000')
SITE_AUTHOR = "Barış Keser"
CONTACT_EMAIL = "info@bariskeser.com"
# CORS Configuration
CORS_ALLOW_ALL_ORIGINS = os.getenv('CORS_ALLOW_ALL_ORIGINS', 'False') == 'True'
SHOPIER_SECRET = os.getenv('SHOPIER_SECRET')
if not SHOPIER_SECRET and not DEBUG:
from django.core.exceptions import ImproperlyConfigured
raise ImproperlyConfigured("SHOPIER_SECRET is not set in production. Billing callbacks will fail.")
# Content Security Policy (CSP)
CSP_DEFAULT_SRC = ("'self'", "cdn.jsdelivr.net", "cdn.tailwindcss.com", "cdn.plyr.io")
CSP_SCRIPT_SRC = ("'self'", "cdn.jsdelivr.net", "cdn.tailwindcss.com", "cdn.plyr.io")
CSP_INCLUDE_NONCE_IN = ('script-src',)
CSP_STYLE_SRC = ("'self'", "'unsafe-inline'", "cdn.jsdelivr.net", "cdn.tailwindcss.com", "cdn.plyr.io", "fonts.googleapis.com")
CSP_IMG_SRC = ("'self'", "data:", "cdn.jsdelivr.net", "cdn.tailwindcss.com", "cdn.plyr.io", "i.ytimg.com", "img.youtube.com")
CSP_FONT_SRC = ("'self'", "fonts.gstatic.com", "cdn.jsdelivr.net")
CSP_MEDIA_SRC = ("'self'", "blob:", "cdn.plyr.io")
CSP_CONNECT_SRC = ("'self'", "ws:", "wss:")
CSP_FRAME_SRC = ("'self'",)
# Unfold Admin Theme Configuration
UNFOLD = {
"DASHBOARD_CALLBACK": "core.dashboard.dashboard_callback",
"SITE_TITLE": "AniScrap Admin",
"SITE_HEADER": "AniScrap Admin",
}
# Default Primary Key Field Type
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
# Auth
LOGIN_URL = '/admin/login/'
LOGIN_REDIRECT_URL = '/profile/'
# REST Framework Configuration
REST_FRAMEWORK = {
'DEFAULT_AUTHENTICATION_CLASSES': (
'rest_framework_simplejwt.authentication.JWTAuthentication',
'rest_framework.authentication.SessionAuthentication',
),
'DEFAULT_PERMISSION_CLASSES': [
'rest_framework.permissions.IsAuthenticatedOrReadOnly',
],
'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination',
'PAGE_SIZE': 20,
'DEFAULT_SCHEMA_CLASS': 'drf_spectacular.openapi.AutoSchema',
'DEFAULT_FILTER_BACKENDS': ['django_filters.rest_framework.DjangoFilterBackend'],
'DEFAULT_THROTTLE_CLASSES': [
'rest_framework.throttling.AnonRateThrottle',
'rest_framework.throttling.UserRateThrottle'
],
'DEFAULT_THROTTLE_RATES': {
'anon': '1000/day',
'user': '10000/day',
'subscribe': '60/minute',
'notifications': '100/minute',
'login': '5/minute',
'watchlog': '10/minute',
'review': '5/hour',
}
}
from datetime import timedelta
SIMPLE_JWT = {
'ACCESS_TOKEN_LIFETIME': timedelta(minutes=60),
'REFRESH_TOKEN_LIFETIME': timedelta(days=7),
'ROTATE_REFRESH_TOKENS': True,
'BLACKLIST_AFTER_ROTATION': True,
}
# Spectacular Configuration
SPECTACULAR_SETTINGS = {
'TITLE': 'AniScrap API',
'DESCRIPTION': 'AniScrap platform API documentation',
'VERSION': '1.0.0',
'SERVE_INCLUDE_SCHEMA': False,
'COMPONENT_SPLIT_REQUEST': True,
}
# CORS Configuration
CORS_ALLOWED_ORIGINS = [
"http://localhost:3000",
"http://localhost:3001",
"http://127.0.0.1:3000",
]
CORS_ALLOW_CREDENTIALS = True