MaxAndEgor/anime-qize
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove Shikimori API, use AnimeThemes only, switch to WebM format

Browse Source 
- Remove ShikimoriService, use AnimeThemes API for search
- Replace shikimori_id with animethemes_slug as primary identifier
- Remove FFmpeg MP3 conversion, download WebM directly
- Add .webm support in storage and upload endpoints
- Update frontend to use animethemes_slug

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
mamonov.ep
2026-01-12 11:22:46 +03:00
parent 333de65fbd
commit cc11f0b773
12 changed files with 138 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
backend/app/main.py
View File

@@ -126,6 +126,7 @@ async def get_audio(filename: str):
".wav": "audio/wav",
".ogg": "audio/ogg",
".m4a": "audio/mp4",
".webm": "video/webm",
}
media_type = media_types.get(suffix, "audio/mpeg")
@@ -284,7 +285,7 @@ async def upload_audio(files: List[UploadFile] = File(...)):
"""Upload audio files to S3."""
results = []
for file in files:
if not file.filename.lower().endswith((".mp3", ".wav", ".ogg", ".m4a")):
if not file.filename.lower().endswith((".mp3", ".wav", ".ogg", ".m4a", ".webm")):
results.append({"filename": file.filename, "success": False, "error": "Invalid file type"})
continue

4
backend/app/openings_downloader/config.py
View File

@@ -4,10 +4,6 @@ from pydantic_settings import BaseSettings
class DownloaderSettings(BaseSettings):
"""Settings for the Openings Downloader module."""
# Shikimori API
shikimori_user_agent: str = "AnimeQuiz/1.0"
shikimori_token: str = "" # Optional OAuth token for higher rate limits
# S3 Storage limit (100 GB default)
s3_storage_limit_bytes: int = 107_374_182_400 # 100 GB

7
backend/app/openings_downloader/db_models.py
View File

@@ -27,12 +27,11 @@ class DownloadStatus(str, enum.Enum):
class Anime(Base):
"""Anime entity from Shikimori."""
"""Anime entity from AnimeThemes."""
__tablename__ = "anime"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
shikimori_id: Mapped[int] = mapped_column(Integer, unique=True, index=True, nullable=False)
animethemes_slug: Mapped[Optional[str]] = mapped_column(String(255), nullable=True, index=True)
animethemes_slug: Mapped[str] = mapped_column(String(255), unique=True, index=True, nullable=False)
title_russian: Mapped[Optional[str]] = mapped_column(String(500), nullable=True)
title_english: Mapped[Optional[str]] = mapped_column(String(500), nullable=True)
@@ -54,7 +53,7 @@ class Anime(Base):
)
def __repr__(self):
return f"<Anime {self.shikimori_id}: {self.title_russian or self.title_english}>"
return f"<Anime {self.animethemes_slug}: {self.title_english}>"
class AnimeTheme(Base):

22
backend/app/openings_downloader/router.py
View File

@@ -21,7 +21,6 @@ from .schemas import (
StorageStatsResponse,
)
from .db_models import Anime, AnimeTheme, DownloadTask, DownloadStatus
from .services.shikimori import ShikimoriService
from .services.animethemes import AnimeThemesService
from .services.downloader import DownloadService
from .services.storage_tracker import StorageTrackerService
@@ -34,30 +33,29 @@ router = APIRouter(prefix="/downloader", tags=["openings-downloader"])
@router.get("/search", response_model=SearchResponse)
async def search_anime(
query: str = Query(..., min_length=1, description="Search query"),
year: Optional[int] = Query(None, description="Filter by year"),
status: Optional[str] = Query(None, description="Filter by status (ongoing, released, announced)"),
limit: int = Query(20, ge=1, le=50, description="Maximum results"),
db: AsyncSession = Depends(get_db),
):
"""Search anime via Shikimori API."""
service = ShikimoriService()
results = await service.search(query, year=year, status=status, limit=limit)
"""Search anime via AnimeThemes API."""
service = AnimeThemesService()
results = await service.search(query, limit=limit)
return SearchResponse(results=results, total=len(results))
# ============== Anime Detail ==============
@router.get("/anime/{shikimori_id}", response_model=AnimeDetailResponse)
@router.get("/anime/{slug:path}", response_model=AnimeDetailResponse)
async def get_anime_detail(
shikimori_id: int,
slug: str,
db: AsyncSession = Depends(get_db),
):
"""Get anime details with available themes from AnimeThemes."""
shikimori_service = ShikimoriService()
animethemes_service = AnimeThemesService()
# Get or create anime record
anime = await shikimori_service.get_or_create_anime(db, shikimori_id)
anime = await animethemes_service.get_or_create_anime(db, slug)
if not anime:
raise HTTPException(status_code=404, detail=f"Anime not found: {slug}")
# Fetch themes from AnimeThemes API
themes = await animethemes_service.fetch_themes(db, anime)
@@ -92,10 +90,8 @@ async def get_anime_detail(
return AnimeDetailResponse(
id=anime.id,
shikimori_id=anime.shikimori_id,
title_russian=anime.title_russian,
animethemes_slug=anime.animethemes_slug,
title_english=anime.title_english,
title_japanese=anime.title_japanese,
year=anime.year,
poster_url=anime.poster_url,
themes=theme_infos,

12
backend/app/openings_downloader/schemas.py
View File

@@ -5,14 +5,12 @@ from pydantic import BaseModel, Field
from .db_models import ThemeType, DownloadStatus
# ============== Shikimori Search ==============
# ============== AnimeThemes Search ==============
class AnimeSearchResult(BaseModel):
"""Single anime search result from Shikimori."""
shikimori_id: int
title_russian: Optional[str] = None
"""Single anime search result from AnimeThemes."""
animethemes_slug: str
title_english: Optional[str] = None
title_japanese: Optional[str] = None
year: Optional[int] = None
poster_url: Optional[str] = None
@@ -48,10 +46,8 @@ class ThemeInfo(BaseModel):
class AnimeDetailResponse(BaseModel):
"""Detailed anime info with themes."""
id: int
shikimori_id: int
title_russian: Optional[str] = None
animethemes_slug: str
title_english: Optional[str] = None
title_japanese: Optional[str] = None
year: Optional[int] = None
poster_url: Optional[str] = None
themes: List[ThemeInfo]

2
backend/app/openings_downloader/services/__init__.py
View File

@@ -1,11 +1,9 @@
# Services for Openings Downloader
from .shikimori import ShikimoriService
from .animethemes import AnimeThemesService
from .downloader import DownloadService
from .storage_tracker import StorageTrackerService
__all__ = [
"ShikimoriService",
"AnimeThemesService",
"DownloadService",
"StorageTrackerService",

131
backend/app/openings_downloader/services/animethemes.py
View File

@@ -7,6 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import selectinload
from ..db_models import Anime, AnimeTheme, ThemeType
from ..schemas import AnimeSearchResult
logger = logging.getLogger(__name__)
@@ -31,41 +32,100 @@ class AnimeThemesService:
def __init__(self):
self.client = _get_animethemes_client()
async def _find_anime_slug(self, anime: Anime) -> Optional[str]:
"""Find AnimeThemes slug by searching anime title."""
async def search(self, query: str, limit: int = 20) -> List[AnimeSearchResult]:
"""Search anime by query using AnimeThemes API."""
try:
response = await self.client.get(
"/anime",
params={
"q": query,
"include": "images",
"page[size]": limit,
},
)
response.raise_for_status()
data = response.json()
# Try different title variations
search_terms = [
anime.title_english,
anime.title_russian,
anime.title_japanese,
]
results = []
for anime in data.get("anime", []):
# Get poster URL from images
poster_url = None
images = anime.get("images", [])
if images:
# Prefer large_cover or first available
for img in images:
if img.get("facet") == "Large Cover":
poster_url = img.get("link")
break
if not poster_url and images:
poster_url = images[0].get("link")
for term in search_terms:
if not term:
continue
results.append(AnimeSearchResult(
animethemes_slug=anime.get("slug"),
title_english=anime.get("name"),
year=anime.get("year"),
poster_url=poster_url,
))
try:
response = await self.client.get(
"/anime",
params={
"q": term,
"include": "animethemes.animethemeentries.videos.audio",
},
)
return results
except Exception as e:
logger.error(f"Failed to search AnimeThemes: {e}")
return []
if response.status_code == 200:
data = response.json()
animes = data.get("anime", [])
if animes:
slug = animes[0].get("slug")
logger.info(f"Found AnimeThemes slug '{slug}' for '{term}'")
return slug
except Exception as e:
logger.warning(f"Failed to search AnimeThemes for '{term}': {e}")
continue
async def get_or_create_anime(self, db: AsyncSession, slug: str) -> Optional[Anime]:
"""Get anime from DB or fetch from AnimeThemes and create."""
return None
# Check if exists (with themes eagerly loaded)
query = (
select(Anime)
.where(Anime.animethemes_slug == slug)
.options(selectinload(Anime.themes))
)
result = await db.execute(query)
anime = result.scalar_one_or_none()
if anime:
return anime
# Fetch from AnimeThemes
try:
response = await self.client.get(
f"/anime/{slug}",
params={"include": "images"},
)
response.raise_for_status()
data = response.json()
except Exception as e:
logger.error(f"Failed to fetch anime from AnimeThemes: {e}")
return None
anime_data = data.get("anime", {})
if not anime_data:
return None
# Get poster URL
poster_url = None
images = anime_data.get("images", [])
if images:
for img in images:
if img.get("facet") == "Large Cover":
poster_url = img.get("link")
break
if not poster_url and images:
poster_url = images[0].get("link")
anime = Anime(
animethemes_slug=slug,
title_english=anime_data.get("name"),
year=anime_data.get("year"),
poster_url=poster_url,
)
db.add(anime)
await db.commit()
await db.refresh(anime)
return anime
async def fetch_themes(self, db: AsyncSession, anime: Anime) -> List[AnimeTheme]:
"""Fetch themes from AnimeThemes API and sync to DB."""
@@ -79,17 +139,8 @@ class AnimeThemesService:
anime = result.scalar_one()
current_themes = anime.themes or []
# Find slug if not cached
if not anime.animethemes_slug:
logger.info(f"Searching AnimeThemes slug for: {anime.title_english or anime.title_russian}")
slug = await self._find_anime_slug(anime)
logger.info(f"Found slug: {slug}")
if slug:
anime.animethemes_slug = slug
await db.commit()
if not anime.animethemes_slug:
logger.warning(f"No AnimeThemes slug found for anime {anime.id}: {anime.title_english or anime.title_russian}")
logger.warning(f"No AnimeThemes slug for anime {anime.id}")
return current_themes
# Fetch themes from AnimeThemes API

36
backend/app/openings_downloader/services/downloader.py
View File

@@ -173,11 +173,10 @@ class DownloadService:
if not theme.animethemes_video_url:
raise ValueError("No video URL available")
# Download and convert in temp directory
# Download WebM file directly (no conversion needed)
with tempfile.TemporaryDirectory() as tmp_dir:
tmp_path = Path(tmp_dir)
webm_file = tmp_path / "video.webm"
mp3_file = tmp_path / "audio.mp3"
webm_file = tmp_path / "audio.webm"
# Stream download WebM file
async with httpx.AsyncClient() as client:
@@ -192,44 +191,23 @@ class DownloadService:
async for chunk in response.aiter_bytes(chunk_size=8192):
f.write(chunk)
task.progress_percent = 40
task.status = DownloadStatus.CONVERTING
await self.db.commit()
# Convert to MP3 with FFmpeg
process = await asyncio.create_subprocess_exec(
"ffmpeg", "-i", str(webm_file),
"-vn", "-acodec", "libmp3lame", "-q:a", "2",
str(mp3_file),
"-y",
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, stderr = await process.communicate()
if process.returncode != 0:
raise RuntimeError(f"FFmpeg error: {stderr.decode()[:500]}")
if not mp3_file.exists():
raise RuntimeError("FFmpeg did not create output file")
task.progress_percent = 70
task.status = DownloadStatus.UPLOADING
await self.db.commit()
# Generate safe S3 key
anime_name = self._sanitize_filename(
anime.title_english or anime.title_russian or f"anime_{anime.shikimori_id}"
anime.title_english or f"anime_{anime.animethemes_slug}"
)
theme_name = theme.full_name
song_part = f"_{self._sanitize_filename(theme.song_title)}" if theme.song_title else ""
s3_key = f"audio/{anime_name}_{theme_name}{song_part}.mp3"
s3_key = f"audio/{anime_name}_{theme_name}{song_part}.webm"
# Read file and upload to S3
file_data = mp3_file.read_bytes()
file_data = webm_file.read_bytes()
file_size = len(file_data)
success = storage.upload_file(s3_key, file_data, "audio/mpeg")
success = storage.upload_file(s3_key, file_data, "video/webm")
if not success:
raise RuntimeError("Failed to upload to S3")
@@ -239,7 +217,7 @@ class DownloadService:
# Create Opening entity in main table
opening = Opening(
anime_name=anime.title_russian or anime.title_english or f"Anime {anime.shikimori_id}",
anime_name=anime.title_english or f"Anime {anime.animethemes_slug}",
op_number=theme_name,
song_name=theme.song_title,
audio_file=s3_key.replace("audio/", ""),

145
backend/app/openings_downloader/services/shikimori.py
View File

@@ -1,145 +0,0 @@
import httpx
from typing import List, Optional
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import selectinload
from ..db_models import Anime
from ..schemas import AnimeSearchResult
from ..config import downloader_settings
# Shared HTTP client for Shikimori API
_shikimori_client: Optional[httpx.AsyncClient] = None
def _get_shikimori_client() -> httpx.AsyncClient:
"""Get or create shared HTTP client for Shikimori."""
global _shikimori_client
if _shikimori_client is None or _shikimori_client.is_closed:
headers = {
"User-Agent": downloader_settings.shikimori_user_agent,
}
if downloader_settings.shikimori_token:
headers["Authorization"] = f"Bearer {downloader_settings.shikimori_token}"
_shikimori_client = httpx.AsyncClient(
headers=headers,
timeout=30.0,
)
return _shikimori_client
class ShikimoriService:
"""Service for Shikimori GraphQL API."""
GRAPHQL_URL = "https://shikimori.one/api/graphql"
def __init__(self):
self.client = _get_shikimori_client()
async def search(
self,
query: str,
year: Optional[int] = None,
status: Optional[str] = None,
limit: int = 20,
) -> List[AnimeSearchResult]:
"""Search anime by query using Shikimori GraphQL API."""
graphql_query = """
query($search: String, $limit: Int, $season: SeasonString, $status: AnimeStatusString) {
animes(search: $search, limit: $limit, season: $season, status: $status) {
id
russian
english
japanese
airedOn { year }
poster { originalUrl }
}
}
"""
variables = {
"search": query,
"limit": limit,
}
if year:
variables["season"] = str(year)
if status:
variables["status"] = status
response = await self.client.post(
self.GRAPHQL_URL,
json={"query": graphql_query, "variables": variables},
)
response.raise_for_status()
data = response.json()
results = []
for anime in data.get("data", {}).get("animes", []):
results.append(AnimeSearchResult(
shikimori_id=int(anime["id"]),
title_russian=anime.get("russian"),
title_english=anime.get("english"),
title_japanese=anime.get("japanese"),
year=anime.get("airedOn", {}).get("year") if anime.get("airedOn") else None,
poster_url=anime.get("poster", {}).get("originalUrl") if anime.get("poster") else None,
))
return results
async def get_or_create_anime(self, db: AsyncSession, shikimori_id: int) -> Anime:
"""Get anime from DB or fetch from Shikimori and create."""
# Check if exists (with themes eagerly loaded)
query = (
select(Anime)
.where(Anime.shikimori_id == shikimori_id)
.options(selectinload(Anime.themes))
)
result = await db.execute(query)
anime = result.scalar_one_or_none()
if anime:
return anime
# Fetch from Shikimori
graphql_query = """
query($ids: String!) {
animes(ids: $ids, limit: 1) {
id
russian
english
japanese
airedOn { year }
poster { originalUrl }
}
}
"""
response = await self.client.post(
self.GRAPHQL_URL,
json={"query": graphql_query, "variables": {"ids": str(shikimori_id)}},
)
response.raise_for_status()
data = response.json()
animes = data.get("data", {}).get("animes", [])
if not animes:
raise ValueError(f"Anime with ID {shikimori_id} not found on Shikimori")
anime_data = animes[0]
anime = Anime(
shikimori_id=shikimori_id,
title_russian=anime_data.get("russian"),
title_english=anime_data.get("english"),
title_japanese=anime_data.get("japanese"),
year=anime_data.get("airedOn", {}).get("year") if anime_data.get("airedOn") else None,
poster_url=anime_data.get("poster", {}).get("originalUrl") if anime_data.get("poster") else None,
)
db.add(anime)
await db.commit()
await db.refresh(anime)
return anime

12
backend/app/storage.py
View File

@@ -117,7 +117,7 @@ class S3Storage:
def list_audio_files(self) -> list[str]:
"""List available audio files."""
return self.list_files("audio/", [".mp3", ".wav", ".ogg", ".m4a"])
return self.list_files("audio/", [".mp3", ".wav", ".ogg", ".m4a", ".webm"])
def list_background_videos(self) -> list[str]:
"""List available background videos."""
@@ -167,7 +167,15 @@ class S3Storage:
def upload_audio(self, filename: str, file_data: bytes) -> bool:
"""Upload audio file to S3."""
content_type = "audio/mpeg" if filename.lower().endswith(".mp3") else "audio/wav"
ext = filename.lower().split(".")[-1]
content_types = {
"mp3": "audio/mpeg",
"wav": "audio/wav",
"ogg": "audio/ogg",
"m4a": "audio/mp4",
"webm": "video/webm",
}
content_type = content_types.get(ext, "audio/mpeg")
return self.upload_file(f"audio/{filename}", file_data, content_type)
def upload_background(self, filename: str, file_data: bytes) -> bool: