diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f0b50fe..d467c69 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,10 @@ Change Log =========== +1.0.7 +------- +- Add markets data to ycnbc + 1.0.6 ------- - Remove pandas from meta.yaml diff --git a/README.md b/README.md index 8746ee3..f75c3da 100644 --- a/README.md +++ b/README.md @@ -35,56 +35,74 @@ $ pip install ycnbc --upgrade --no-cache-dir --- -### Usage +### Usage for Markets +```python +import ycnbc + +markets = ycnbc.Markets() + +quote_summary = markets.quote_summary('AAPL') +pre_markets = markets.pre_markets() +us_markets = markets.us_markets() +europe_markets = markets.europe_markets() +asia_markets = markets.asia_markets() +currencies = markets.currencies() +cryptocurrencies = markets.cryptocurrencies() +futures_and_commodities = markets.futures_and_commodities() +bonds = markets.bonds() +funds_and_etfs = markets.funds_and_etfs() +``` + +### Usage for news ```python import ycnbc -data = ycnbc.News() +news = ycnbc.News() # Get trending news -trending_ = data.trending() +trending_ = news.trending() # Get latest news -latest_ = data.latest() +latest_ = news.latest() # Get news by category -economy_ = data.economy() -jobs_ = data.jobs() -white_house_ = data.white_house() -hospitals_ = data.hospitals() -transportation_ = data.transportation() -media_ = data.media() -internet_ = data.internet() -congress_ = data.congress() -policy_ = data.policy() -finance_ = data.finance() -life_ = data.life() -defense_ = data.defense() -europe_politics_ = data.europe_politics() -china_politics_ = data.china_politics() -asia_politics_ = data.asia_politics() -world_politics_ = data.world_politics() -equity_opportunity_ = data.equity_opportunity() -politics_ = data.politics() -wealth_ = data.wealth() -world_economy_ = data.world_economy() -central_banks_ = data.central_banks() -real_estate_ = data.real_estate() -health_science_ = data.health_science() -small_business_ = data.small_business() -lifehealth_insurance_ = data.lifehealth_insurance() -business_ = data.business() -energy_ = data.energy() -industrials_ = data.industrials() -retail_ = data.retail() -cybersecurity_ = data.cybersecurity() -mobile_ = data.mobile() -technology_ = data.technology() -cnbc_disruptors_ = data.cnbc_disruptors() -tech_guide_ = data.tech_guide() -social_media_ = data.social_media() -climate_ = data.climate() +economy_ = news.economy() +jobs_ = news.jobs() +white_house_ = news.white_house() +hospitals_ = news.hospitals() +transportation_ = news.transportation() +media_ = news.media() +internet_ = news.internet() +congress_ = news.congress() +policy_ = news.policy() +finance_ = news.finance() +life_ = news.life() +defense_ = news.defense() +europe_politics_ = news.europe_politics() +china_politics_ = news.china_politics() +asia_politics_ = news.asia_politics() +world_politics_ = news.world_politics() +equity_opportunity_ = news.equity_opportunity() +politics_ = news.politics() +wealth_ = news.wealth() +world_economy_ = news.world_economy() +central_banks_ = news.central_banks() +real_estate_ = news.real_estate() +health_science_ = news.health_science() +small_business_ = news.small_business() +lifehealth_insurance_ = news.lifehealth_insurance() +business_ = news.business() +energy_ = news.energy() +industrials_ = news.industrials() +retail_ = news.retail() +cybersecurity_ = news.cybersecurity() +mobile_ = news.mobile() +technology_ = news.technology() +cnbc_disruptors_ = news.cnbc_disruptors() +tech_guide_ = news.tech_guide() +social_media_ = news.social_media() +climate_ = news.climate() ``` Note: diff --git a/meta.yaml b/meta.yaml index 960f86a..7e84ec4 100644 --- a/meta.yaml +++ b/meta.yaml @@ -1,5 +1,5 @@ {% set name = "ycnbc" %} -{% set version = "1.0.6" %} +{% set version = "1.0.7" %} package: name: "{{ name|lower }}" @@ -7,7 +7,7 @@ package: source: url: "https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name }}-{{ version }}.tar.gz" - sha256: "901BC86660D303E1C6533D96FA41687E623DD7682429C2225747E5CEFBDFB120" + sha256: "670D54B9688CC95B4776CF887CFA2E723D642A7A174B86F33B0D629868A1A39C" build: noarch: python diff --git a/test.py b/test.py index f493f96..abfe693 100644 --- a/test.py +++ b/test.py @@ -1,10 +1,14 @@ import ycnbc import unittest - -data = ycnbc.News() +import json class TestData(unittest.TestCase): + + def setUp(self): + self.markets = ycnbc.Markets() + self.news = ycnbc.News() + def test_cnbc_news(self): methods = [ 'latest', @@ -49,10 +53,34 @@ def test_cnbc_news(self): for method_name in methods: with self.subTest(method=method_name): - method = getattr(data, method_name) + method = getattr(self.news, method_name) response = method() self.assertNotIn("error", response, f"{method_name} returned an error") + def test_cnbc_markets(self): + quote_summary = self.markets.quote_summary('AAPL') + pre_markets = self.markets.pre_markets() + us_markets = self.markets.us_markets() + europe_markets = self.markets.europe_markets() + asia_markets = self.markets.asia_markets() + currencies = self.markets.currencies() + cryptocurrencies = self.markets.cryptocurrencies() + futures_and_commodities = self.markets.futures_and_commodities() + bonds = self.markets.bonds() + funds_and_etfs = self.markets.funds_and_etfs() + + print(json.dumps(funds_and_etfs)) + self.assertIsNotNone(quote_summary) + self.assertIsNotNone(pre_markets) + self.assertIsNotNone(us_markets) + self.assertIsNotNone(europe_markets) + self.assertIsNotNone(asia_markets) + self.assertIsNotNone(currencies) + self.assertIsNotNone(cryptocurrencies) + self.assertIsNotNone(futures_and_commodities) + self.assertIsNotNone(bonds) + self.assertIsNotNone(funds_and_etfs) + if __name__ == '__main__': unittest.main() diff --git a/ycnbc/__init__.py b/ycnbc/__init__.py index b57d35d..6228e93 100644 --- a/ycnbc/__init__.py +++ b/ycnbc/__init__.py @@ -20,10 +20,11 @@ # from . import version -from .base import News +from .news import News +from .markets import Markets __version__ = version.version __author__ = "Asep Saputra" -__all__ = ['News'] +__all__ = [News, Markets] diff --git a/ycnbc/markets/__init__.py b/ycnbc/markets/__init__.py new file mode 100644 index 0000000..49e643d --- /dev/null +++ b/ycnbc/markets/__init__.py @@ -0,0 +1 @@ +from .markets_base import Markets diff --git a/ycnbc/markets/markets_base.py b/ycnbc/markets/markets_base.py new file mode 100644 index 0000000..00eb08f --- /dev/null +++ b/ycnbc/markets/markets_base.py @@ -0,0 +1,67 @@ +from .markets_utils import MarketUtils + + +class Markets: + + def __init__(self): + self.market_utils = MarketUtils() + + def quote_summary(self, symbol): + result = self.market_utils.fetch_data(symbol) + if result: + return result[0] + return None + + def pre_markets(self): + symbols = (".HSI|.N225|.STI|.AXJO|.SSEC|.FTSE|.GDAXI|.FCHI|.AEX|.STOXX50|@CL.1|@RB.1|@NG.1|@GC.1|@SI.1|EUR" + "=|JPY=|GBP=|CHF=|CAD=|.VIX|.VXN|.OVX|.OOI|.FTFCNBCG|US3M|US2Y|US5Y|US10Y|US30Y|.RUT|.DJT|.DJU" + "|.NDX|.NYA|.GSPT|.GSPS|.GSPD|.GSPE|.GSPF") + return self.market_utils.fetch_data(symbols) + + def us_markets(self): + symbols = (".DJI|.IXIC|.SPX|@GC.1|@CL.1|US10Y|EUR=|.VIX|.DJT|.DJU|.NDX|.MID|.RUT|.NYA|@RB.1|@NG.1|@HO.1|@SI.1" + "|@HG.1|@W.1|@S.1|@C.1|US1M|US3M|US6M|US1Y|US2Y|US30Y|.DXY|JPY=|GBP=|CAD=|AUD=|SEK=|CHF=") + return self.market_utils.fetch_data(symbols) + + def europe_markets(self): + symbols = (".FTSE|.GDAXI|.FCHI|.STOXX|.AEX|.BFX|.FTMIB|.OMXS30|.SSMI|.OMXHPI|.PSI20|.OMXC25CAP|.IMOEX|EUR=|GBP" + "=|EURJPY=|EURGBP=|CHF=|EURAUD=|EURCHF=|EURCAD=|UK10Y-GB|DE10Y-DE|IT10Y-IT|FR10Y-FR|ES10Y-ES") + return self.market_utils.fetch_data(symbols) + + def asia_markets(self): + symbols = (".SSEC|.N225|.HSI|.SZI|.KS11|.AXJO|.NSEI|.STI|.IECNCGP|.FTFCNBCA|.NZ50|.SETI|.KLSE|.TWII|JPY" + "=|EURJPY=|AUD=|CNY=|SGD=|NZD=|HKD=|INR=|@CL.1|@LCO.1|@NG.1|@HG.1") + return self.market_utils.fetch_data(symbols) + + def currencies(self): + symbols = ("EUR=|JPY=|GBP=|CAD=|CHF=|AUD=|EURCHF=|EURGBP=|" + "EURJPY=|AUDJPY=|MXN=|BRL=|ARS=|COP=|CLP=|NZD=|" + "GD=|HKD=|KRW=|INR=|SEK=|NOK=|RUB=|DKK=|PLN=") + return self.market_utils.fetch_data(symbols) + + def cryptocurrencies(self): + symbols = ("BTC.CM=|BTC.BS=|BTC.CB=|BTC.GM=|BTC.BF=|BCH.CM=|BCH.BS=|BCH.CB=" + "BAT.CM=|BNB.CM=|EOS.CM=|ETH.CM=|ETC.CM=|ETH=|ETH.GM=|ETH.BF=" + "LTC.CM=|LTC.BF=|LTC.CB=|LUNA.CM=|XRP.CM=|XRP.BS=|XTZ.CM=|XLM.CM=" + "XLM.BF=|ZRX.CM=|ZRX.BF=|ZEC.CM=|DOGE.CM=|NEXO.CM=|ADA.CM=|LINK.CM=" + "MATIC.CM=|ALGO.CM=|ATOM.CM=|SOL.CM=|DOT.CM=|AMP.CM=|SAND.CM=" + "AXS.CM=|CRO.CM=|BUSD.CM=|USDC.CM=|UST.CM=|USDT.CM=|UNI.CM=" + "DAI.CM=|SHIB.CM=|MANA.CM=|TRX.CM=") + return self.market_utils.fetch_data(symbols) + + def futures_and_commodities(self): + symbols = ("@CL.1|@LCO.1|@NG.1|@RB.1|@GC.1|@SI.1|@PL.1|@HG.1|@PA.1|@LBR.1|@W.1|@S.1|@C.1|@SB.1|@KC.1|@CT.1" + "|@RR.1|@CC.1|@DX.1|@URO.1|@JY.1|@BP.1|@XBT.1|@LC.1|@LH.1|@FC.1|@DJ.1|@SP.1|@ND.1|@MD.1|@US.1|@TY" + ".1|@FV.1|@TU.1|@GE.1") + return self.market_utils.fetch_data(symbols) + + def bonds(self): + symbols = ("US1M|US2M|US3M|US4M|US6M|US1Y|US2Y|US3Y|US5Y|US7Y|US10Y|US20Y|US30Y|UK2Y-GB|UK5Y-GB|UK10Y-GB" + "|UK30Y-GB|DE10Y-DE|DE20Y-DE|DE30Y-DE") + return self.market_utils.fetch_data(symbols) + + def funds_and_etfs(self): + symbols = ( + "TLT|SPY|QQQ|IWM|DIA|MDY|EEM|EFA|IVV|VTI|IJR|SSO|TQQQ|SQQQ|EDC|XLE|XLF|XLU|XLI|XLK|XLV|XLP|XLY|GDX|IYR" + "|GLD|SLV|UNG|USO|IAU|DBC|UUP|FXE|FXY|FXB|EUO|YCS") + return self.market_utils.fetch_data(symbols) diff --git a/ycnbc/markets/markets_utils.py b/ycnbc/markets/markets_utils.py new file mode 100644 index 0000000..a8b472c --- /dev/null +++ b/ycnbc/markets/markets_utils.py @@ -0,0 +1,39 @@ +import requests + + +class MarketUtils: + def __init__(self): + self.base_url = "https://quote.cnbc.com/quote-html-webservice/restQuote/symbolType/symbol" + + def fetch_data(self, symbols): + params = { + 'symbols': symbols, + 'requestMethod': 'itv', + 'noform': '1', + 'partnerId': '2', + 'fund': '1', + 'exthrs': '1', + 'output': 'json', + 'events': '1' + } + + try: + response = requests.get(self.base_url, params=params) + if response.status_code == 200: + data = response.json() + formatted_quote = data.get('FormattedQuoteResult', {}).get('FormattedQuote', []) + + if formatted_quote: + for item in formatted_quote: + keys_to_remove = [ + 'portfolioindicator', 'feedSymbol', 'issue_id', + 'streamable', 'provider', 'code', 'symbolType', 'altSymbol' + ] + for key in keys_to_remove: + if key in item: + del item[key] + return formatted_quote + return None + except Exception as e: + print(f"An error occurred: {e}") + return None diff --git a/ycnbc/news/__init__.py b/ycnbc/news/__init__.py new file mode 100644 index 0000000..83b568d --- /dev/null +++ b/ycnbc/news/__init__.py @@ -0,0 +1 @@ +from .news_base import News diff --git a/ycnbc/base.py b/ycnbc/news/news_base.py similarity index 79% rename from ycnbc/base.py rename to ycnbc/news/news_base.py index ad3e315..5ad3bd7 100644 --- a/ycnbc/base.py +++ b/ycnbc/news/news_base.py @@ -1,30 +1,9 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# ycnbc - CNBC data downloader -# https://github.com/asepscareer/ycnbc -# -# Copyright 2022 Asep Saputra -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from .utils import CNBCNews +from .news_utils import CNBCNewsUtils class News: def __init__(self): - self.news = CNBCNews() + self.news = CNBCNewsUtils() def latest(self): return self.news.latest() diff --git a/ycnbc/utils.py b/ycnbc/news/news_utils.py similarity index 78% rename from ycnbc/utils.py rename to ycnbc/news/news_utils.py index dd0592f..e006a2a 100644 --- a/ycnbc/utils.py +++ b/ycnbc/news/news_utils.py @@ -1,30 +1,11 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# ycnbc - CNBC data downloader -# https://github.com/asepscareer/ycnbc -# -# Copyright 2022 Asep Saputra -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# from __future__ import print_function from lxml import html from requests import get -from .uri import _BASE_URL_, _HEADERS_ +from .uri import _HEADERS_, _BASE_URL_ -class CNBCNews: + +class CNBCNewsUtils: def __init__(self): self.base_url = _BASE_URL_ self.headers = _HEADERS_ @@ -42,7 +23,7 @@ def _fetch_page(self, endpoint=""): try: url = f"{self.base_url}/{endpoint}" if endpoint else self.base_url page = get(url, headers=self.headers) - page.raise_for_status() # Ensure we raise an error for bad HTTP responses + page.raise_for_status() return html.fromstring(page.content) except Exception as e: return {"error": str(e)} @@ -71,8 +52,8 @@ def trending(self): source.append(link) return { - 'Title': title, - 'Link': source + 'title': title, + 'link': source } except Exception as e: return {"error": str(e)} @@ -113,9 +94,9 @@ def latest(self): posttime.append(' '.join(posttime_)) return { - 'Headline': title, - 'Post Time': posttime, - 'Link': source + 'headline': title, + 'time': posttime, + 'link': source } except Exception as e: return {"error": str(e)} @@ -156,9 +137,9 @@ def by_category(self, category): title.append(' '.join(text)) return { - 'Headline': title, - 'Post Time': posttime, - 'Link': source + 'headline': title, + 'time': posttime, + 'link': source } except Exception as e: return {"error": str(e)} diff --git a/ycnbc/news/uri.py b/ycnbc/news/uri.py new file mode 100644 index 0000000..9795244 --- /dev/null +++ b/ycnbc/news/uri.py @@ -0,0 +1,9 @@ +_BASE_URL_ = 'https://www.cnbc.com' +_HEADERS_ = { + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,' + 'application/signed-exchange;v=b3', + 'Accept-Encoding': 'gzip, deflate, br', + 'DNT': '1', + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/39.0.2171.95 Safari/537.36' +} \ No newline at end of file diff --git a/ycnbc/uri.py b/ycnbc/uri.py deleted file mode 100644 index a522230..0000000 --- a/ycnbc/uri.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# ycnbc - CNBC data downloader -# https://github.com/asepscareer/ycnbc -# -# Copyright 2022 Asep Saputra -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -_BASE_URL_ = 'https://www.cnbc.com' -_HEADERS_ = { - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,' - 'application/signed-exchange;v=b3', - 'Accept-Encoding': 'gzip, deflate, br', - 'DNT': '1', - 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) ' - 'Chrome/39.0.2171.95 Safari/537.36' -} \ No newline at end of file diff --git a/ycnbc/version.py b/ycnbc/version.py index 8f63b1f..ac8c2aa 100644 --- a/ycnbc/version.py +++ b/ycnbc/version.py @@ -1 +1 @@ -version = "1.0.6" +version = "1.0.7"