[25.06.07 / TASK-206] Feature - 사용자 모델 (유저네임, 썸네일) 추가, 대응 개발 (#31)

* modify: username 배치 대응 개발, 세부 업데이트
* feature: test code fixture 분리와 userinfo 업데이트 테스트 로직 추가
* modify: update_old_user_info 좀 더 파이써닉 (walrus operator) 하게 업데이트

Author: Nuung

common/models.py

@@ -1,6 +1,6 @@
 import json
 from dataclasses import dataclass
-from typing import Type, TypeVar
+from typing import Any, Type, TypeVar, no_type_check
 
 from django.db import models
 
@@ -31,13 +31,16 @@ class Meta:
 # dataclass 베이스 mixin
 @dataclass
 class SerializableMixin:
-    def to_dict(self) -> dict:
+    @no_type_check
+    def to_dict(self) -> dict[str, Any]:
         return to_dict(self)
 
-    def to_json_dict(self) -> dict:
+    @no_type_check
+    def to_json_dict(self) -> dict[str, Any]:
         """Django Model의 JSON 필드 저장용"""
         return json.loads(json.dumps(self.to_dict()))
 
     @classmethod
-    def from_dict(cls: Type[T], data: dict) -> T:
+    @no_type_check
+    def from_dict(cls: Type[T], data: dict[str, Any]) -> T:
         return from_dict(cls, data)

scraping/apis.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Any
 
 from aiohttp.client import ClientSession
 from aiohttp_retry import ExponentialRetry, RetryClient
@@ -27,7 +28,7 @@ async def fetch_velog_user_chk(
     session: ClientSession,
     access_token: str,
     refresh_token: str,
-) -> tuple[dict[str, str], dict[str, str]]:
+) -> tuple[dict[str, Any], dict[str, Any]]:
     # 토큰 유효성 검증
     payload = {"query": CURRENT_USER_QUERY}
     headers = get_header(access_token, refresh_token)

scraping/constants.py

@@ -21,6 +21,9 @@
             id
             username
             email
+            profile {
+                thumbnail
+            }
         }
     }
     """

scraping/main.py

@@ -1,5 +1,6 @@
 import asyncio
 import logging
+from typing import Any
 
 import aiohttp
 import async_timeout
@@ -72,10 +73,61 @@ async def update_old_tokens(
             sentry_sdk.capture_exception(e)
         return False
 
+    async def update_old_user_info(
+        self, user: User, user_data: dict[str, Any]
+    ) -> bool:
+        """사용자 프로필 정보 업데이트"""
+
+        field_updates = {}
+
+        # 각 필드별 업데이트 체크 및 적용
+        if (new_email := user_data.get("email")) and (
+            not user.email or user.email != new_email
+        ):
+            field_updates["email"] = new_email
+
+        if (new_username := user_data.get("username")) and (
+            not user.username or user.username != new_username
+        ):
+            field_updates["username"] = new_username
+
+        if (profile := user_data.get("profile")) and (
+            new_thumbnail := profile.get("thumbnail")
+        ):
+            if not user.thumbnail or user.thumbnail != new_thumbnail:
+                field_updates["thumbnail"] = new_thumbnail
+
+        # 업데이트할 필드가 없으면 조기 반환
+        if not field_updates:
+            return True
+
+        try:
+            # 필드 일괄 업데이트
+            for field, value in field_updates.items():
+                setattr(user, field, value)
+
+            await user.asave(update_fields=list(field_updates.keys()))
+
+            logger.info(
+                "Updated user profile fields %s for %s",
+                list(field_updates.keys()),
+                user.velog_uuid,
+            )
+            return True
+
+        except Exception as e:
+            logger.error(
+                "Failed to update user info: %s (user velog uuid: %s)",
+                e,
+                user.velog_uuid,
+            )
+            sentry_sdk.capture_exception(e)
+        return False
+
     async def bulk_upsert_posts(
         self,
         user: User,
-        fetched_posts: list[dict[str, str]],
+        fetched_posts: list[dict[str, Any]],
         batch_size: int = 200,
     ) -> bool:
         """Post 객체를 일정 크기의 배치로 나눠서 삽입 또는 업데이트"""
@@ -95,7 +147,7 @@ async def bulk_upsert_posts(
             return False
 
     async def _upsert_batch(
-        self, user: User, batch_posts: list[dict[str, str]]
+        self, user: User, batch_posts: list[dict[str, Any]]
     ) -> None:
         """단일 배치 처리, bulk_upsert_posts 에서 호출됨"""
 
@@ -209,7 +261,7 @@ def _execute_sync() -> None:
         await _execute_sync()
 
     async def update_daily_statistics(
-        self, post: dict[str, str], stats: dict[str, str]
+        self, post: dict[str, Any], stats: dict[str, Any]
     ) -> None:
         """PostDailyStatistics를 업데이트 또는 생성 (upsert)"""
         if not stats or not isinstance(stats, dict):
@@ -222,17 +274,17 @@ async def update_daily_statistics(
             today = get_local_now().date()
             post_id = post["id"]
 
-            stats_data = stats.get("data", {})  # type: ignore
+            stats_data = stats.get("data", {})
             if not stats_data or not isinstance(
-                stats_data.get("getStats"),  # type: ignore
+                stats_data.get("getStats"),
                 dict,
             ):
                 logger.warning(
                     f"Skip updating statistics due to missing getStats data for post {post_id}"
                 )
                 return
 
-            view_count = stats_data["getStats"].get("total", 0)  # type: ignore
+            view_count = stats_data["getStats"].get("total", 0)
             like_count = post.get("likes", 0)
 
             # 트랜잭션 내에서 실행
@@ -331,29 +383,50 @@ async def process_user(
         origin_access_token = aes_encryption.decrypt(user.access_token)
         origin_refresh_token = aes_encryption.decrypt(user.refresh_token)
 
+        # ========================================================== #
+        # STEP1: 토큰이 유효성 체크 및 업데이트. 이후 사용자 정보 업데이트
+        # ========================================================== #
+
         # 토큰 유효성 검증
         new_user_cookies, user_data = await fetch_velog_user_chk(
             session,
             origin_access_token,
             origin_refresh_token,
         )
-        if not (user_data or new_user_cookies):
-            return
 
-        if user_data["data"]["currentUser"] is None:  # type: ignore
+        if (
+            not (user_data or new_user_cookies)
+            or user_data.get("data", {}).get("currentUser") is None
+        ):
             logger.warning(
                 f"Failed to fetch user data because of wrong tokens. (user velog uuid: {user.velog_uuid})"
             )
             return
 
         if new_user_cookies:
-            await self.update_old_tokens(
+            user_token_result = await self.update_old_tokens(
                 user,
                 aes_encryption,
                 new_user_cookies,
             )
+            if not user_token_result:
+                raise Exception("Failed to update tokens, Check the logs")
+            origin_access_token = new_user_cookies["access_token"]
+            origin_refresh_token = new_user_cookies["refresh_token"]
+
+        # velog 응답과 기존 저장된 사용자 정보 비교 및 업데이트
+        # user_data -> currentUser 에는 id / username / email / profile { thumbnail } 존재
+        user_info_result = await self.update_old_user_info(
+            user,
+            user_data["data"]["currentUser"],
+        )
+        if not user_info_result:
+            raise Exception("Failed to update user_info, Check the logs")
 
-        username = user_data["data"]["currentUser"]["username"]  # type: ignore
+        # ========================================================== #
+        # STEP2: 게시물 전체 목록을 가져와서 upsert 와 상태 동기화 (비활성, 활성)
+        # ========================================================== #
+        username = user_data["data"]["currentUser"]["username"]
         fetched_posts = await fetch_all_velog_posts(
             session, username, origin_access_token, origin_refresh_token
         )
@@ -370,6 +443,9 @@ async def process_user(
             user, all_post_ids, min_posts_threshold=1
         )
 
+        # ========================================================== #
+        # STEP3: 게시물 전체 목록을 기반으로 세부 통계 가져와서 upsert
+        # ========================================================== #
         # 게시물을 적절한 크기의 청크로 나누어 처리
         chunk_size = 20
         for i in range(0, len(fetched_posts), chunk_size):

scraping/tests/conftest.py

@@ -0,0 +1,78 @@
+import uuid
+
+import pytest
+
+from scraping.main import Scraper
+from users.models import User
+
+
+@pytest.fixture
+def scraper():
+    """Scraper 인스턴스 생성"""
+    return Scraper(group_range=range(1, 10), max_connections=10)
+
+
+@pytest.fixture
+def user(db):
+    """테스트용 User 객체 생성"""
+    return User.objects.create(
+        velog_uuid=uuid.uuid4(),
+        access_token="encrypted-access-token",
+        refresh_token="encrypted-refresh-token",
+        group_id=1,
+        email="test@example.com",
+        username="nuung",
+        thumbnail="https://nuung.com",
+        is_active=True,
+    )
+
+
+@pytest.fixture
+def mock_user_data():
+    """테스트용 user_data 구조"""
+    return {
+        "data": {
+            "currentUser": {
+                "id": "user-123",
+                "email": "test@example.com",
+                "username": "testuser",
+                "profile": {"thumbnail": "https://example.com/thumbnail.jpg"},
+            }
+        }
+    }
+
+
+@pytest.fixture
+def mock_new_tokens():
+    """테스트용 새 토큰"""
+    return {
+        "access_token": "new-access-token",
+        "refresh_token": "new-refresh-token",
+    }
+
+
+@pytest.fixture
+def mock_posts_data():
+    """테스트용 게시물 데이터"""
+    return [
+        {
+            "id": str(uuid.uuid4()),
+            "title": "Test Post 1",
+            "url_slug": "test-post-1",
+            "released_at": "2024-01-01T00:00:00Z",
+            "likes": 15,
+        },
+        {
+            "id": str(uuid.uuid4()),
+            "title": "Test Post 2",
+            "url_slug": "test-post-2",
+            "released_at": "2024-01-02T00:00:00Z",
+            "likes": 25,
+        },
+    ]
+
+
+@pytest.fixture
+def mock_stats_data():
+    """테스트용 통계 데이터"""
+    return {"data": {"getStats": {"total": 150}}}

scraping/tests/test_main_posts.py

@@ -5,29 +5,11 @@
 from asgiref.sync import sync_to_async
 
 from posts.models import Post
-from scraping.main import Scraper
 from users.models import User
 from utils.utils import get_local_now
 
 
 class TestScraperPosts:
-    @pytest.fixture
-    def scraper(self):
-        """Scraper 인스턴스 생성"""
-        return Scraper(group_range=range(1, 10), max_connections=10)
-
-    @pytest.fixture
-    def user(self, db):
-        """테스트용 User 객체 생성"""
-        return User.objects.create(
-            velog_uuid=uuid.uuid4(),
-            access_token="encrypted-access-token",
-            refresh_token="encrypted-refresh-token",
-            group_id=1,
-            email="test@example.com",
-            is_active=True,
-        )
-
     @pytest.mark.asyncio
     async def test_bulk_upsert_posts_success(self, scraper, user):
         """Post 객체 배치 분할 삽입 또는 업데이트 성공 테스트"""

scraping/tests/test_main_statistics.py

@@ -5,29 +5,11 @@
 from asgiref.sync import sync_to_async
 
 from posts.models import Post, PostDailyStatistics
-from scraping.main import Scraper
 from users.models import User
 from utils.utils import get_local_now
 
 
 class TestScraperStatistics:
-    @pytest.fixture
-    def scraper(self):
-        """Scraper 인스턴스 생성"""
-        return Scraper(group_range=range(1, 10), max_connections=10)
-
-    @pytest.fixture
-    def user(self, db):
-        """테스트용 User 객체 생성"""
-        return User.objects.create(
-            velog_uuid=uuid.uuid4(),
-            access_token="encrypted-access-token",
-            refresh_token="encrypted-refresh-token",
-            group_id=1,
-            email="test@example.com",
-            is_active=True,
-        )
-
     @pytest.mark.asyncio
     async def test_update_daily_statistics_success(self, scraper):
         """데일리 통계 업데이트 또는 생성 성공 테스트"""