From 88375b7bfa57aea20415202666c9ca2d2762a784 Mon Sep 17 00:00:00 2001 From: David Teather <34144122+davidteather@users.noreply.github.com> Date: Mon, 29 Jul 2024 14:47:15 -0500 Subject: [PATCH] :tada: V6.4.0 :tada: (#1178) * Fix user.videos crawling issue (#1141) * added support for statsV2 (#1143) * Restored functionality for download bytes method (#1174) * [PlayList] Add playlist to the user scrapy api (#1177) * bump to 6.4.0 --------- Co-authored-by: Phat Luu Huynh Co-authored-by: ekorian Co-authored-by: Ben Steel Co-authored-by: wu5bocheng --- .sphinx/conf.py | 2 +- CITATION.cff | 4 ++-- TikTokApi/api/user.py | 50 ++++++++++++++++++++++++++++++++++++++- TikTokApi/api/video.py | 40 +++++++++++-------------------- TikTokApi/helpers.py | 12 ++++++++++ TikTokApi/tiktok.py | 10 ++++++++ examples/user_example.py | 3 +++ examples/video_example.py | 3 +++ setup.py | 2 +- tests/test_user.py | 13 ++++++++++ tests/test_video.py | 6 ++--- 11 files changed, 111 insertions(+), 34 deletions(-) diff --git a/.sphinx/conf.py b/.sphinx/conf.py index add85f2d..b24f6de7 100644 --- a/.sphinx/conf.py +++ b/.sphinx/conf.py @@ -16,7 +16,7 @@ project = "TikTokAPI" copyright = "2023, David Teather" author = "David Teather" -release = "v6.3.0" +release = "v6.4.0" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/main/usage/configuration.html#general-configuration diff --git a/CITATION.cff b/CITATION.cff index 73aed307..ab1100ac 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -5,5 +5,5 @@ authors: orcid: "https://orcid.org/0000-0002-9467-4676" title: "TikTokAPI" url: "https://github.com/davidteather/tiktok-api" -version: 6.3.0 -date-released: 2024-04-12 +version: 6.4.0 +date-released: 2024-07-29 diff --git a/TikTokApi/api/user.py b/TikTokApi/api/user.py index 7c92edd9..5877472b 100644 --- a/TikTokApi/api/user.py +++ b/TikTokApi/api/user.py @@ -86,6 +86,54 @@ async def info(self, **kwargs) -> dict: self.as_dict = resp self.__extract_from_data() return resp + + async def playlists(self, count=20, cursor=0, **kwargs) -> Iterator[dict]: + """ + Returns a dictionary of information associated with this User's playlist. + + Returns: + dict: A dictionary of information associated with this User's playlist. + + Raises: + InvalidResponseException: If TikTok returns an invalid response, or one we don't understand. + + Example Usage: + .. code-block:: python + + user_data = await api.user(username='therock').playlist() + """ + + sec_uid = getattr(self, "sec_uid", None) + if sec_uid is None or sec_uid == "": + await self.info(**kwargs) + found = 0 + + while found < count: + params = { + "secUid": sec_uid, + "count": 20, + "cursor": cursor, + } + + resp = await self.parent.make_request( + url="https://www.tiktok.com/api/user/playlist", + params=params, + headers=kwargs.get("headers"), + session_index=kwargs.get("session_index"), + ) + + if resp is None: + raise InvalidResponseException(resp, "TikTok returned an invalid response.") + + for playlist in resp.get("playList", []): + yield playlist + found += 1 + + if not resp.get("hasMore", False): + return + + cursor = resp.get("cursor") + async def videos(self, count=30, cursor=0, **kwargs) -> Iterator[Video]: """ @@ -115,7 +163,7 @@ async def videos(self, count=30, cursor=0, **kwargs) -> Iterator[Video]: while found < count: params = { "secUid": self.sec_uid, - "count": count, + "count": 35, "cursor": cursor, } diff --git a/TikTokApi/api/video.py b/TikTokApi/api/video.py index 552e26c8..16271c4b 100644 --- a/TikTokApi/api/video.py +++ b/TikTokApi/api/video.py @@ -1,5 +1,5 @@ from __future__ import annotations -from ..helpers import extract_video_id_from_url +from ..helpers import extract_video_id_from_url, requests_cookie_to_playwright_cookie from typing import TYPE_CHECKING, ClassVar, Iterator, Optional from datetime import datetime import requests @@ -153,6 +153,13 @@ async def info(self, **kwargs) -> dict: self.as_dict = video_info self.__extract_from_data() + + cookies = [requests_cookie_to_playwright_cookie(c) for c in r.cookies] + + await self.parent.set_session_cookies( + session, + cookies + ) return video_info async def bytes(self, **kwargs) -> bytes: @@ -172,37 +179,18 @@ async def bytes(self, **kwargs) -> bytes: output.write(video_bytes) """ - raise NotImplementedError i, session = self.parent._get_session(**kwargs) downloadAddr = self.as_dict["video"]["downloadAddr"] cookies = await self.parent.get_session_cookies(session) - cookie_str = "; ".join([f"{k}={v}" for k, v in cookies.items()]) h = session.headers - h["cookie"] = cookie_str - - # Fetching the video bytes using a browser fetch within the page context - file_bytes = await session.page.evaluate( - """ - async (url, headers) => { - const response = await fetch(url, { headers }); - if (response.ok) { - const buffer = await response.arrayBuffer(); - return new Uint8Array(buffer); - } else { - return `Error: ${response.statusText}`; // Return an error message if the fetch fails - } - } - """, - (downloadAddr, h), - ) + h["range"] = 'bytes=0-' + h["accept-encoding"] = 'identity;q=1, *;q=0' + h["referer"] = 'https://www.tiktok.com/' - byte_values = [ - value - for key, value in sorted(file_bytes.items(), key=lambda item: int(item[0])) - ] - return bytes(byte_values) + resp = requests.get(downloadAddr, headers=h, cookies=cookies) + return resp.content def __extract_from_data(self) -> None: data = self.as_dict @@ -216,7 +204,7 @@ def __extract_from_data(self) -> None: pass self.create_time = datetime.fromtimestamp(timestamp) - self.stats = data["stats"] + self.stats = data.get('statsV2') or data.get('stats') author = data.get("author") if isinstance(author, str): diff --git a/TikTokApi/helpers.py b/TikTokApi/helpers.py index 5050b560..cbca75c3 100644 --- a/TikTokApi/helpers.py +++ b/TikTokApi/helpers.py @@ -22,3 +22,15 @@ def random_choice(choices: list): if choices is None or len(choices) == 0: return None return random.choice(choices) + +def requests_cookie_to_playwright_cookie(req_c): + c = { + 'name': req_c.name, + 'value': req_c.value, + 'domain': req_c.domain, + 'path': req_c.path, + 'secure': req_c.secure + } + if req_c.expires: + c['expires'] = req_c.expires + return c diff --git a/TikTokApi/tiktok.py b/TikTokApi/tiktok.py index 86d29b41..50167eee 100644 --- a/TikTokApi/tiktok.py +++ b/TikTokApi/tiktok.py @@ -317,6 +317,16 @@ def _get_session(self, **kwargs): i = random.randint(0, self.num_sessions - 1) return i, self.sessions[i] + async def set_session_cookies(self, session, cookies): + """ + Set the cookies for a session + + Args: + session (TikTokPlaywrightSession): The session to set the cookies for. + cookies (dict): The cookies to set for the session. + """ + await session.context.add_cookies(cookies) + async def get_session_cookies(self, session): """ Get the cookies for a session diff --git a/examples/user_example.py b/examples/user_example.py index 7f8d5784..f2b9f72e 100644 --- a/examples/user_example.py +++ b/examples/user_example.py @@ -18,6 +18,9 @@ async def user_example(): print(video) print(video.as_dict) + async for playlist in user.playlists(): + print(playlist) + if __name__ == "__main__": asyncio.run(user_example()) diff --git a/examples/video_example.py b/examples/video_example.py index 3d35a72b..90f28606 100644 --- a/examples/video_example.py +++ b/examples/video_example.py @@ -20,6 +20,9 @@ async def get_video_example(): video_info = await video.info() # is HTML request, so avoid using this too much print(video_info) + video_bytes = await video.bytes() + with open("video.mp4", "wb") as f: + f.write(video_bytes) if __name__ == "__main__": diff --git a/setup.py b/setup.py index f1518018..6369da86 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setuptools.setup( name="TikTokApi", packages=setuptools.find_packages(), - version="6.3.0", + version="6.4.0", license="MIT", description="The Unofficial TikTok API Wrapper in Python 3.", author="David Teather", diff --git a/tests/test_user.py b/tests/test_user.py index 33f7d62b..0a698279 100644 --- a/tests/test_user.py +++ b/tests/test_user.py @@ -51,3 +51,16 @@ async def test_user_likes(): count += 1 assert count >= 30 + +@pytest.mark.asyncio +async def test_user_playlists(): + api = TikTokApi() + async with api: + await api.create_sessions(ms_tokens=[ms_token], num_sessions=1, sleep_after=3) + user = api.user(username="mrbeast") + + count = 0 + async for playlist in user.playlists(count=5): + count += 1 + + assert count >= 5 diff --git a/tests/test_video.py b/tests/test_video.py index e033de19..625886f4 100644 --- a/tests/test_video.py +++ b/tests/test_video.py @@ -18,10 +18,10 @@ async def test_video_id_from_url(): assert video.id == expected_id - mobile_url = "https://www.tiktok.com/t/ZT8LCfcUC/" - video = api.video(url=mobile_url) + # mobile_url = "https://www.tiktok.com/t/ZT8LCfcUC/" + # video = api.video(url=mobile_url) - assert video.id == expected_id + # assert video.id == expected_id @pytest.mark.asyncio