Remove Shikimori API, use AnimeThemes only, switch to WebM format
- Remove ShikimoriService, use AnimeThemes API for search - Replace shikimori_id with animethemes_slug as primary identifier - Remove FFmpeg MP3 conversion, download WebM directly - Add .webm support in storage and upload endpoints - Update frontend to use animethemes_slug 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -1,11 +1,9 @@
|
||||
# Services for Openings Downloader
|
||||
from .shikimori import ShikimoriService
|
||||
from .animethemes import AnimeThemesService
|
||||
from .downloader import DownloadService
|
||||
from .storage_tracker import StorageTrackerService
|
||||
|
||||
__all__ = [
|
||||
"ShikimoriService",
|
||||
"AnimeThemesService",
|
||||
"DownloadService",
|
||||
"StorageTrackerService",
|
||||
|
||||
@@ -7,6 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from ..db_models import Anime, AnimeTheme, ThemeType
|
||||
from ..schemas import AnimeSearchResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -31,41 +32,100 @@ class AnimeThemesService:
|
||||
def __init__(self):
|
||||
self.client = _get_animethemes_client()
|
||||
|
||||
async def _find_anime_slug(self, anime: Anime) -> Optional[str]:
|
||||
"""Find AnimeThemes slug by searching anime title."""
|
||||
async def search(self, query: str, limit: int = 20) -> List[AnimeSearchResult]:
|
||||
"""Search anime by query using AnimeThemes API."""
|
||||
try:
|
||||
response = await self.client.get(
|
||||
"/anime",
|
||||
params={
|
||||
"q": query,
|
||||
"include": "images",
|
||||
"page[size]": limit,
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
# Try different title variations
|
||||
search_terms = [
|
||||
anime.title_english,
|
||||
anime.title_russian,
|
||||
anime.title_japanese,
|
||||
]
|
||||
results = []
|
||||
for anime in data.get("anime", []):
|
||||
# Get poster URL from images
|
||||
poster_url = None
|
||||
images = anime.get("images", [])
|
||||
if images:
|
||||
# Prefer large_cover or first available
|
||||
for img in images:
|
||||
if img.get("facet") == "Large Cover":
|
||||
poster_url = img.get("link")
|
||||
break
|
||||
if not poster_url and images:
|
||||
poster_url = images[0].get("link")
|
||||
|
||||
for term in search_terms:
|
||||
if not term:
|
||||
continue
|
||||
results.append(AnimeSearchResult(
|
||||
animethemes_slug=anime.get("slug"),
|
||||
title_english=anime.get("name"),
|
||||
year=anime.get("year"),
|
||||
poster_url=poster_url,
|
||||
))
|
||||
|
||||
try:
|
||||
response = await self.client.get(
|
||||
"/anime",
|
||||
params={
|
||||
"q": term,
|
||||
"include": "animethemes.animethemeentries.videos.audio",
|
||||
},
|
||||
)
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to search AnimeThemes: {e}")
|
||||
return []
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
animes = data.get("anime", [])
|
||||
if animes:
|
||||
slug = animes[0].get("slug")
|
||||
logger.info(f"Found AnimeThemes slug '{slug}' for '{term}'")
|
||||
return slug
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to search AnimeThemes for '{term}': {e}")
|
||||
continue
|
||||
async def get_or_create_anime(self, db: AsyncSession, slug: str) -> Optional[Anime]:
|
||||
"""Get anime from DB or fetch from AnimeThemes and create."""
|
||||
|
||||
return None
|
||||
# Check if exists (with themes eagerly loaded)
|
||||
query = (
|
||||
select(Anime)
|
||||
.where(Anime.animethemes_slug == slug)
|
||||
.options(selectinload(Anime.themes))
|
||||
)
|
||||
result = await db.execute(query)
|
||||
anime = result.scalar_one_or_none()
|
||||
|
||||
if anime:
|
||||
return anime
|
||||
|
||||
# Fetch from AnimeThemes
|
||||
try:
|
||||
response = await self.client.get(
|
||||
f"/anime/{slug}",
|
||||
params={"include": "images"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch anime from AnimeThemes: {e}")
|
||||
return None
|
||||
|
||||
anime_data = data.get("anime", {})
|
||||
if not anime_data:
|
||||
return None
|
||||
|
||||
# Get poster URL
|
||||
poster_url = None
|
||||
images = anime_data.get("images", [])
|
||||
if images:
|
||||
for img in images:
|
||||
if img.get("facet") == "Large Cover":
|
||||
poster_url = img.get("link")
|
||||
break
|
||||
if not poster_url and images:
|
||||
poster_url = images[0].get("link")
|
||||
|
||||
anime = Anime(
|
||||
animethemes_slug=slug,
|
||||
title_english=anime_data.get("name"),
|
||||
year=anime_data.get("year"),
|
||||
poster_url=poster_url,
|
||||
)
|
||||
|
||||
db.add(anime)
|
||||
await db.commit()
|
||||
await db.refresh(anime)
|
||||
|
||||
return anime
|
||||
|
||||
async def fetch_themes(self, db: AsyncSession, anime: Anime) -> List[AnimeTheme]:
|
||||
"""Fetch themes from AnimeThemes API and sync to DB."""
|
||||
@@ -79,17 +139,8 @@ class AnimeThemesService:
|
||||
anime = result.scalar_one()
|
||||
current_themes = anime.themes or []
|
||||
|
||||
# Find slug if not cached
|
||||
if not anime.animethemes_slug:
|
||||
logger.info(f"Searching AnimeThemes slug for: {anime.title_english or anime.title_russian}")
|
||||
slug = await self._find_anime_slug(anime)
|
||||
logger.info(f"Found slug: {slug}")
|
||||
if slug:
|
||||
anime.animethemes_slug = slug
|
||||
await db.commit()
|
||||
|
||||
if not anime.animethemes_slug:
|
||||
logger.warning(f"No AnimeThemes slug found for anime {anime.id}: {anime.title_english or anime.title_russian}")
|
||||
logger.warning(f"No AnimeThemes slug for anime {anime.id}")
|
||||
return current_themes
|
||||
|
||||
# Fetch themes from AnimeThemes API
|
||||
|
||||
@@ -173,11 +173,10 @@ class DownloadService:
|
||||
if not theme.animethemes_video_url:
|
||||
raise ValueError("No video URL available")
|
||||
|
||||
# Download and convert in temp directory
|
||||
# Download WebM file directly (no conversion needed)
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
tmp_path = Path(tmp_dir)
|
||||
webm_file = tmp_path / "video.webm"
|
||||
mp3_file = tmp_path / "audio.mp3"
|
||||
webm_file = tmp_path / "audio.webm"
|
||||
|
||||
# Stream download WebM file
|
||||
async with httpx.AsyncClient() as client:
|
||||
@@ -192,44 +191,23 @@ class DownloadService:
|
||||
async for chunk in response.aiter_bytes(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
|
||||
task.progress_percent = 40
|
||||
task.status = DownloadStatus.CONVERTING
|
||||
await self.db.commit()
|
||||
|
||||
# Convert to MP3 with FFmpeg
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
"ffmpeg", "-i", str(webm_file),
|
||||
"-vn", "-acodec", "libmp3lame", "-q:a", "2",
|
||||
str(mp3_file),
|
||||
"-y",
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = await process.communicate()
|
||||
|
||||
if process.returncode != 0:
|
||||
raise RuntimeError(f"FFmpeg error: {stderr.decode()[:500]}")
|
||||
|
||||
if not mp3_file.exists():
|
||||
raise RuntimeError("FFmpeg did not create output file")
|
||||
|
||||
task.progress_percent = 70
|
||||
task.status = DownloadStatus.UPLOADING
|
||||
await self.db.commit()
|
||||
|
||||
# Generate safe S3 key
|
||||
anime_name = self._sanitize_filename(
|
||||
anime.title_english or anime.title_russian or f"anime_{anime.shikimori_id}"
|
||||
anime.title_english or f"anime_{anime.animethemes_slug}"
|
||||
)
|
||||
theme_name = theme.full_name
|
||||
song_part = f"_{self._sanitize_filename(theme.song_title)}" if theme.song_title else ""
|
||||
s3_key = f"audio/{anime_name}_{theme_name}{song_part}.mp3"
|
||||
s3_key = f"audio/{anime_name}_{theme_name}{song_part}.webm"
|
||||
|
||||
# Read file and upload to S3
|
||||
file_data = mp3_file.read_bytes()
|
||||
file_data = webm_file.read_bytes()
|
||||
file_size = len(file_data)
|
||||
|
||||
success = storage.upload_file(s3_key, file_data, "audio/mpeg")
|
||||
success = storage.upload_file(s3_key, file_data, "video/webm")
|
||||
if not success:
|
||||
raise RuntimeError("Failed to upload to S3")
|
||||
|
||||
@@ -239,7 +217,7 @@ class DownloadService:
|
||||
|
||||
# Create Opening entity in main table
|
||||
opening = Opening(
|
||||
anime_name=anime.title_russian or anime.title_english or f"Anime {anime.shikimori_id}",
|
||||
anime_name=anime.title_english or f"Anime {anime.animethemes_slug}",
|
||||
op_number=theme_name,
|
||||
song_name=theme.song_title,
|
||||
audio_file=s3_key.replace("audio/", ""),
|
||||
|
||||
@@ -1,145 +0,0 @@
|
||||
import httpx
|
||||
from typing import List, Optional
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from ..db_models import Anime
|
||||
from ..schemas import AnimeSearchResult
|
||||
from ..config import downloader_settings
|
||||
|
||||
# Shared HTTP client for Shikimori API
|
||||
_shikimori_client: Optional[httpx.AsyncClient] = None
|
||||
|
||||
|
||||
def _get_shikimori_client() -> httpx.AsyncClient:
|
||||
"""Get or create shared HTTP client for Shikimori."""
|
||||
global _shikimori_client
|
||||
if _shikimori_client is None or _shikimori_client.is_closed:
|
||||
headers = {
|
||||
"User-Agent": downloader_settings.shikimori_user_agent,
|
||||
}
|
||||
if downloader_settings.shikimori_token:
|
||||
headers["Authorization"] = f"Bearer {downloader_settings.shikimori_token}"
|
||||
_shikimori_client = httpx.AsyncClient(
|
||||
headers=headers,
|
||||
timeout=30.0,
|
||||
)
|
||||
return _shikimori_client
|
||||
|
||||
|
||||
class ShikimoriService:
|
||||
"""Service for Shikimori GraphQL API."""
|
||||
|
||||
GRAPHQL_URL = "https://shikimori.one/api/graphql"
|
||||
|
||||
def __init__(self):
|
||||
self.client = _get_shikimori_client()
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
year: Optional[int] = None,
|
||||
status: Optional[str] = None,
|
||||
limit: int = 20,
|
||||
) -> List[AnimeSearchResult]:
|
||||
"""Search anime by query using Shikimori GraphQL API."""
|
||||
|
||||
graphql_query = """
|
||||
query($search: String, $limit: Int, $season: SeasonString, $status: AnimeStatusString) {
|
||||
animes(search: $search, limit: $limit, season: $season, status: $status) {
|
||||
id
|
||||
russian
|
||||
english
|
||||
japanese
|
||||
airedOn { year }
|
||||
poster { originalUrl }
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
variables = {
|
||||
"search": query,
|
||||
"limit": limit,
|
||||
}
|
||||
if year:
|
||||
variables["season"] = str(year)
|
||||
if status:
|
||||
variables["status"] = status
|
||||
|
||||
response = await self.client.post(
|
||||
self.GRAPHQL_URL,
|
||||
json={"query": graphql_query, "variables": variables},
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
results = []
|
||||
for anime in data.get("data", {}).get("animes", []):
|
||||
results.append(AnimeSearchResult(
|
||||
shikimori_id=int(anime["id"]),
|
||||
title_russian=anime.get("russian"),
|
||||
title_english=anime.get("english"),
|
||||
title_japanese=anime.get("japanese"),
|
||||
year=anime.get("airedOn", {}).get("year") if anime.get("airedOn") else None,
|
||||
poster_url=anime.get("poster", {}).get("originalUrl") if anime.get("poster") else None,
|
||||
))
|
||||
|
||||
return results
|
||||
|
||||
async def get_or_create_anime(self, db: AsyncSession, shikimori_id: int) -> Anime:
|
||||
"""Get anime from DB or fetch from Shikimori and create."""
|
||||
|
||||
# Check if exists (with themes eagerly loaded)
|
||||
query = (
|
||||
select(Anime)
|
||||
.where(Anime.shikimori_id == shikimori_id)
|
||||
.options(selectinload(Anime.themes))
|
||||
)
|
||||
result = await db.execute(query)
|
||||
anime = result.scalar_one_or_none()
|
||||
|
||||
if anime:
|
||||
return anime
|
||||
|
||||
# Fetch from Shikimori
|
||||
graphql_query = """
|
||||
query($ids: String!) {
|
||||
animes(ids: $ids, limit: 1) {
|
||||
id
|
||||
russian
|
||||
english
|
||||
japanese
|
||||
airedOn { year }
|
||||
poster { originalUrl }
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
response = await self.client.post(
|
||||
self.GRAPHQL_URL,
|
||||
json={"query": graphql_query, "variables": {"ids": str(shikimori_id)}},
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
animes = data.get("data", {}).get("animes", [])
|
||||
if not animes:
|
||||
raise ValueError(f"Anime with ID {shikimori_id} not found on Shikimori")
|
||||
|
||||
anime_data = animes[0]
|
||||
|
||||
anime = Anime(
|
||||
shikimori_id=shikimori_id,
|
||||
title_russian=anime_data.get("russian"),
|
||||
title_english=anime_data.get("english"),
|
||||
title_japanese=anime_data.get("japanese"),
|
||||
year=anime_data.get("airedOn", {}).get("year") if anime_data.get("airedOn") else None,
|
||||
poster_url=anime_data.get("poster", {}).get("originalUrl") if anime_data.get("poster") else None,
|
||||
)
|
||||
|
||||
db.add(anime)
|
||||
await db.commit()
|
||||
await db.refresh(anime)
|
||||
|
||||
return anime
|
||||
Reference in New Issue
Block a user