Merge : API 이슈 수정 확인 배포

.github/workflows/update-ky-youtube.yml

@@ -0,0 +1,62 @@
+name: Update ky by Youtube
+
+on:
+  schedule:
+    - cron: "0 14 * * *" # 한국 시간 23:00 실행 (UTC+9 → UTC 14:00)
+  workflow_dispatch:
+
+permissions:
+  contents: write # push 권한을 위해 필요
+
+jobs:
+  run-npm-task:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout branch
+        uses: actions/checkout@v4
+        with:
+          ref: feat/songUpdate
+          persist-credentials: false # 수동 인증으로 푸시 제어
+
+      - name: Use Node.js 18
+        uses: actions/setup-node@v4
+        with:
+          node-version: "18"
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2
+        with:
+          version: 9
+          run_install: false
+
+      - name: Install dependencies
+        working-directory: packages/crawling
+        run: pnpm install
+
+      - name: Create .env file
+        working-directory: packages/crawling
+        run: |
+          echo "SUPABASE_URL=${{ secrets.SUPABASE_URL }}" >> .env
+          echo "SUPABASE_KEY=${{ secrets.SUPABASE_KEY }}" >> .env
+
+      - name: run update script - packages/crawling/crawlYoutube.ts
+        working-directory: packages/crawling
+        run: pnpm run ky-youtube
+
+      - name: Commit and push changes to feat/songUpdate branch
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+          git checkout feat/songUpdate
+
+          git add .
+          if git diff --cached --quiet; then
+            echo "✅ No changes to commit"
+          else
+            git commit -m "chore: update crawled TJ song data [skip ci]"
+            git push origin feat/songUpdate
+          fi
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

apps/web/src/hooks/useSearchSong.ts

@@ -25,10 +25,10 @@ export default function useSearchSong() {
   const [saveModalType, setSaveModalType] = useState<SaveModalType>('');
   const [selectedSaveSong, setSelectedSaveSong] = useState<SearchSong | null>(null);
   // const { data: searchResults, isLoading } = useSearchSongQuery(query, searchType, isAuthenticated);
-  const { mutate: toggleToSing } = useToggleToSingMutation();
-  const { mutate: toggleLike } = useToggleLikeMutation();
-  const { mutate: postSong } = useSaveMutation();
-  const { mutate: moveSong } = useMoveSaveSongMutation();
+  const { mutate: toggleToSing, isPending: isToggleToSingPending } = useToggleToSingMutation();
+  const { mutate: toggleLike, isPending: isToggleLikePending } = useToggleLikeMutation();
+  const { mutate: postSong, isPending: isPostSongPending } = useSaveMutation();
+  const { mutate: moveSong, isPending: isMoveSongPending } = useMoveSaveSongMutation();
 
   const {
     data: searchResults,
@@ -56,6 +56,11 @@ export default function useSearchSong() {
       toast.error('로그인이 필요해요.');
       return;
     }
+
+    if (isToggleToSingPending) {
+      toast.error('요청 중입니다. 잠시 후 다시 시도해주세요.');
+      return;
+    }
     toggleToSing({ songId, method, query, searchType });
   };
 
@@ -64,6 +69,11 @@ export default function useSearchSong() {
       toast.error('로그인이 필요해요.');
       return;
     }
+
+    if (isToggleLikePending) {
+      toast.error('요청 중입니다. 잠시 후 다시 시도해주세요.');
+      return;
+    }
     toggleLike({ songId, method, query, searchType });
   };
 
@@ -72,15 +82,25 @@ export default function useSearchSong() {
       toast.error('로그인이 필요해요.');
       return;
     }
+
     setSelectedSaveSong(song);
     setSaveModalType(method === 'POST' ? 'POST' : 'PATCH');
   };
 
   const postSaveSong = async (songId: string, folderName: string) => {
+    if (isPostSongPending) {
+      toast.error('요청 중입니다. 잠시 후 다시 시도해주세요.');
+      return;
+    }
     postSong({ songId, folderName, query, searchType });
   };
 
   const patchSaveSong = async (songId: string, folderId: string) => {
+    if (isMoveSongPending) {
+      toast.error('요청 중입니다. 잠시 후 다시 시도해주세요.');
+      return;
+    }
+
     moveSong({ songIdArray: [songId], folderId });
   };
 

apps/web/src/queries/tosingSongQuery.ts

@@ -102,35 +102,6 @@ export function useDeleteToSingSongMutation() {
   });
 }
 
-// 여러 곡 부를 노래 삭제 - 미사용?
-
-// export function useDeleteToSingSongArrayMutation() {
-//   const queryClient = useQueryClient();
-
-//   return useMutation({
-//     mutationFn: (songIds: string[]) => deleteToSingSongArray({ songIds }),
-//     onMutate: async (songIds: string[]) => {
-//       queryClient.cancelQueries({ queryKey: ['toSingSong'] });
-//       const prev = queryClient.getQueryData(['toSingSong']);
-//       queryClient.setQueryData(['toSingSong'], (old: ToSingSong[]) =>
-//         old.filter(song => !songIds.includes(song.songs.id)),
-//       );
-//       return { prev };
-//     },
-//     onError: (error, variables, context) => {
-//       console.error('error', error);
-//       alert(error.message ?? 'DELETE 실패');
-//       queryClient.setQueryData(['toSingSong'], context?.prev);
-//     },
-//     onSettled: () => {
-//       queryClient.invalidateQueries({ queryKey: ['toSingSong'] });
-//       queryClient.invalidateQueries({ queryKey: ['likeSong'] });
-//       queryClient.invalidateQueries({ queryKey: ['saveSongFolder'] });
-//       queryClient.invalidateQueries({ queryKey: ['recentSingLog'] });
-//     },
-//   });
-// }
-
 // 🎵 부를 노래 순서 변경
 export function usePatchToSingSongMutation() {
   const queryClient = useQueryClient();

packages/crawling/package.json

@@ -9,7 +9,7 @@
   "scripts": {
     "ky-open": "tsx src/findKYByOpen.ts",
     "ky-youtube": "tsx src/crawling/crawlYoutube.ts",
-    "ky-valid": "tsx src/crawling/crawlYoutubeValid.ts",
+    "ky-youtube-ubuntu": "tsx src/crawling/crawlYoutubeUbuntu.ts",
     "ky-update": "pnpm run ky-youtube & pnpm run ky-valid",
     "trans": "tsx src/postTransDictionary.ts",
     "recent-tj": "tsx src/crawling/crawlRecentTJ.ts",

packages/crawling/src/assets/crawlKYYoutubeFailedList.txt

@@ -25805,3 +25805,4 @@ Nightwalker-텐(TEN)
 My Love Mine All Mine-Mitski
 MONSTERS(최강야구OST)-이원석
 Lose Control-Teddy Swims
+Fruit Fly-Leah Dou,검정치마

packages/crawling/src/crawling/crawlRecentTJ.ts

@@ -15,6 +15,7 @@ dotenv.config();
 
 // action 우분투 환경에서의 호환을 위해 추가
 const browser = await puppeteer.launch({
+  headless: true,
   args: ['--no-sandbox', '--disable-setuid-sandbox'],
 });
 

packages/crawling/src/crawling/crawlYoutube.ts

@@ -15,7 +15,11 @@ import { isValidKYExistNumber } from './isValidKYExistNumber';
 // youtube에서 KY 노래방 번호 크롤링
 // crawlYoutubeValid에서 진행하는 실제 사이트 검증도 포함
 
-const browser = await puppeteer.launch();
+// action 우분투 환경에서의 호환을 위해 추가
+const browser = await puppeteer.launch({
+  headless: true,
+});
+
 const page = await browser.newPage();
 
 const baseUrl = 'https://www.youtube.com/@KARAOKEKY/search';
@@ -62,17 +66,22 @@ const data = await getSongsKyNullDB();
 const failedSongs = loadCrawlYoutubeFailedKYSongs();
 
 console.log('getSongsKyNullDB : ', data.length);
+console.log(failedSongs.size);
 let index = 0;
 
 for (const song of data) {
+  // 테스트를 위해 100회 반복 후 종료시키기
+  if (index >= 100) {
+    break;
+  }
+
   const query = song.title + '-' + song.artist;
 
   if (failedSongs.has(query)) {
+    console.log('failedSongs has : ', query);
     continue;
   }
 
-  console.log(song.title, ' - ', song.artist);
-
   let resultKyNum = null;
   try {
     resultKyNum = await scrapeSongNumber(query);
@@ -97,6 +106,7 @@ for (const song of data) {
   } else saveCrawlYoutubeFailedKYSongs(song.title, song.artist);
 
   index++;
+  console.log(query);
   console.log('scrapeSongNumber : ', index);
 }
 

packages/crawling/src/crawling/crawlYoutubeUbuntu.ts

@@ -0,0 +1,127 @@
+import * as cheerio from 'cheerio';
+import puppeteer from 'puppeteer';
+
+import { getSongsKyNullDB } from '@/supabase/getDB';
+import { updateSongsKyDB } from '@/supabase/updateDB';
+import { Song } from '@/types';
+import {
+  loadCrawlYoutubeFailedKYSongs,
+  saveCrawlYoutubeFailedKYSongs,
+  updateDataLog,
+} from '@/utils/logData';
+
+import { isValidKYExistNumber } from './isValidKYExistNumber';
+
+// youtube에서 KY 노래방 번호 크롤링
+// crawlYoutubeValid에서 진행하는 실제 사이트 검증도 포함
+
+// action 우분투 환경에서의 호환을 위해 추가
+// const browser = await puppeteer.launch({
+//   headless: true,
+//   executablePath: '/usr/bin/chromium-browser', // 또는 "/usr/bin/chromium"
+//   args: [
+//     '--no-sandbox',
+//     '--disable-setuid-sandbox',
+//     '--disable-dev-shm-usage', // 리눅스 메모리 제한 대응
+//     '--disable-gpu',
+//     '--disable-infobars',
+//     '--single-process',
+//     '--window-size=1920,1080',
+//   ],
+// });
+
+const browser = await puppeteer.launch({
+  headless: true,
+  args: ['--no-sandbox', '--disable-setuid-sandbox'],
+});
+
+const page = await browser.newPage();
+
+const baseUrl = 'https://www.youtube.com/@KARAOKEKY/search';
+
+const scrapeSongNumber = async (query: string) => {
+  const searchUrl = `${baseUrl}?query=${encodeURIComponent(query)}`;
+
+  // page.goto의 waitUntil 문제였음!
+  await page.goto(searchUrl, {
+    waitUntil: 'networkidle2',
+    timeout: 0,
+  });
+
+  const html = await page.content();
+  const $ = cheerio.load(html);
+
+  // id contents 의 첫번째  ytd-item-section-renderer 찾기
+  // const firstItem = $("#contents ytd-item-section-renderer").first();
+
+  const firstItem = $('ytd-video-renderer').first();
+
+  // yt-formatted-string 찾기
+  const title = firstItem.find('yt-formatted-string').first().text().trim();
+
+  const karaokeNumber = extractKaraokeNumber(title);
+
+  return karaokeNumber;
+};
+
+const extractKaraokeNumber = (title: string) => {
+  // KY. 찾고 ) 가 올때까지 찾기
+  const matchResult = title.match(/KY\.\s*(\d{2,5})\)/);
+  const karaokeNumber = matchResult ? matchResult[1] : null;
+  return karaokeNumber;
+};
+
+const updateData = async (data: Song) => {
+  const result = await updateSongsKyDB(data);
+  updateDataLog(result.success, 'crawlYoutubeSuccess.txt');
+  updateDataLog(result.failed, 'crawlYoutubeFailed.txt');
+};
+
+const data = await getSongsKyNullDB();
+const failedSongs = loadCrawlYoutubeFailedKYSongs();
+
+console.log('getSongsKyNullDB : ', data.length);
+let index = 0;
+
+for (const song of data) {
+  // 테스트를 위해 100회 반복 후 종료시키기
+  if (index >= 100) {
+    break;
+  }
+
+  const query = song.title + '-' + song.artist;
+
+  if (failedSongs.has(query)) {
+    continue;
+  }
+
+  console.log(song.title, ' - ', song.artist);
+
+  let resultKyNum = null;
+  try {
+    resultKyNum = await scrapeSongNumber(query);
+  } catch (error) {
+    continue;
+  }
+
+  if (resultKyNum) {
+    let isValid = true;
+    try {
+      isValid = await isValidKYExistNumber(page, resultKyNum, song.title, song.artist);
+    } catch (error) {
+      continue;
+    }
+
+    if (!isValid) {
+      saveCrawlYoutubeFailedKYSongs(song.title, song.artist);
+      continue;
+    } else {
+      await updateData({ ...song, num_ky: resultKyNum });
+    }
+  } else saveCrawlYoutubeFailedKYSongs(song.title, song.artist);
+
+  index++;
+  console.log('scrapeSongNumber : ', index);
+}
+
+browser.close();