replace requests with NetworkClient in content/

kolibri/core/content/api.py

@@ -5,7 +5,6 @@
 from functools import reduce
 from random import sample
 
-import requests
 from django.core.cache import cache
 from django.core.exceptions import ValidationError
 from django.db.models import Exists
@@ -32,7 +31,6 @@
 from django_filters.rest_framework import UUIDFilter
 from le_utils.constants import content_kinds
 from le_utils.constants import languages
-from requests.exceptions import RequestException
 from rest_framework import filters
 from rest_framework import mixins
 from rest_framework import status
@@ -72,12 +70,14 @@
     get_channel_stats_from_studio,
 )
 from kolibri.core.content.utils.paths import get_channel_lookup_url
-from kolibri.core.content.utils.paths import get_info_url
 from kolibri.core.content.utils.paths import get_local_content_storage_file_url
 from kolibri.core.content.utils.search import get_available_metadata_labels
 from kolibri.core.content.utils.stopwords import stopwords_set
 from kolibri.core.decorators import query_params_required
 from kolibri.core.device.models import ContentCacheKey
+from kolibri.core.discovery.utils.network.client import NetworkClient
+from kolibri.core.discovery.utils.network.errors import NetworkLocationConnectionFailure
+from kolibri.core.discovery.utils.network.errors import NetworkLocationResponseFailure
 from kolibri.core.discovery.utils.network.errors import ResourceGoneError
 from kolibri.core.lessons.models import Lesson
 from kolibri.core.logger.models import ContentSessionLog
@@ -86,7 +86,7 @@
 from kolibri.core.utils.pagination import ValuesViewsetCursorPagination
 from kolibri.core.utils.pagination import ValuesViewsetLimitOffsetPagination
 from kolibri.core.utils.pagination import ValuesViewsetPageNumberPagination
-from kolibri.core.utils.urls import join_url
+from kolibri.utils import conf
 from kolibri.utils.conf import OPTIONS
 from kolibri.utils.urls import validator
 
@@ -210,14 +210,13 @@ def _hande_proxied_request(self, request):
             validator(baseurl)
         except ValidationError:
             raise Http404("Remote resource not found")
-        remote_url = join_url(baseurl, remote_path)
+        client = NetworkClient.build_for_address(baseurl)
+        remote_url = remote_path
         try:
-            response = requests.get(
+            response = client.get(
                 remote_url, params=qs, headers=self._get_request_headers(request)
             )
-            if response.status_code == 404:
-                raise Http404("Remote resource not found")
-            response.raise_for_status()
+
             # If Etag is set on the response we have returned here, any further Etag will not be modified
             # by the django etag decorator, so this should allow us to transparently proxy the remote etag.
             try:
@@ -231,8 +230,9 @@ def _hande_proxied_request(self, request):
                 status=response.status_code,
                 headers=headers,
             )
-        except RequestException:
-            # If any sort of error due to connection or timeout, raise a resource gone error
+        except NetworkLocationResponseFailure as e:
+            if e.response.status_code == 404:
+                raise Http404("Remote resource not found")
             raise ResourceGoneError
 
 
@@ -1744,23 +1744,26 @@ def _make_channel_endpoint_request(
         if baseurl is not None:
             try:
                 validator(baseurl)
+                client = NetworkClient.build_for_address(baseurl)
             except ValidationError:
                 baseurl = None
+        if baseurl is None:
+            client = NetworkClient("/")
         url = get_channel_lookup_url(
             identifier=identifier, baseurl=baseurl, keyword=keyword, language=language
         )
+        try:
 
-        resp = requests.get(url)
-
-        if resp.status_code == 404:
-            raise Http404(
-                _("The requested channel does not exist on the content server")
-            )
-
-        # map the channel list into the format the Kolibri client-side expects
-        channels = list(map(self._studio_response_to_kolibri_response, resp.json()))
+            resp = client.get(url)
+            # map the channel list into the format the Kolibri client-side expects
+            channels = list(map(self._studio_response_to_kolibri_response, resp.json()))
 
-        return channels
+            return channels
+        except NetworkLocationResponseFailure as e:
+            if e.response.status_code == 404:
+                raise Http404(
+                    _("The requested channel does not exist on the content server")
+                )
 
     @staticmethod
     def _get_lang_native_name(code):
@@ -1824,7 +1827,7 @@ def list(self, request, *args, **kwargs):
             channels = self._make_channel_endpoint_request(
                 identifier=token, baseurl=baseurl, keyword=keyword, language=language
             )
-        except requests.exceptions.ConnectionError:
+        except NetworkLocationConnectionFailure:
             return Response(
                 {"status": "offline"}, status=status.HTTP_503_SERVICE_UNAVAILABLE
             )
@@ -1841,7 +1844,7 @@ def retrieve(self, request, pk=None):
             channels = self._make_channel_endpoint_request(
                 identifier=pk, baseurl=baseurl, keyword=keyword, language=language
             )
-        except requests.exceptions.ConnectionError:
+        except NetworkLocationConnectionFailure:
             return Response(
                 {"status": "offline"}, status=status.HTTP_503_SERVICE_UNAVAILABLE
             )
@@ -1853,13 +1856,12 @@ def retrieve(self, request, pk=None):
     @no_cache_on_method
     def kolibri_studio_status(self, request, **kwargs):
         try:
-            resp = requests.get(get_info_url())
-            if resp.status_code == 404:
-                raise requests.ConnectionError("Kolibri Studio URL is incorrect!")
-            else:
-                data = resp.json()
-                data["available"] = True
-                data["status"] = "online"
-                return Response(data)
-        except requests.ConnectionError:
+            baseurl = conf.OPTIONS["Urls"]["CENTRAL_CONTENT_BASE_URL"]
+            client = NetworkClient.build_for_address(baseurl)
+            resp = client.get("/api/public/info")
+            data = resp.json()
+            data["available"] = True
+            data["status"] = "online"
+            return Response(data)
+        except (NetworkLocationResponseFailure, NetworkLocationConnectionFailure):
             return Response({"status": "offline", "available": False})

kolibri/core/content/tasks.py

@@ -1,6 +1,3 @@
-from urllib.parse import urljoin
-
-import requests
 from django.core.exceptions import ValidationError
 from django.core.management import call_command
 from django.db.models import Q
@@ -33,6 +30,7 @@
 from kolibri.core.discovery.utils.network.client import NetworkClient
 from kolibri.core.discovery.utils.network.errors import IncompatibleVersionError
 from kolibri.core.discovery.utils.network.errors import NetworkLocationNotFound
+from kolibri.core.discovery.utils.network.errors import NetworkLocationResponseFailure
 from kolibri.core.discovery.utils.network.errors import ResourceGoneError
 from kolibri.core.serializers import HexOnlyUUIDField
 from kolibri.core.tasks.decorators import register_task
@@ -43,7 +41,7 @@
 from kolibri.core.tasks.permissions import CanManageContent
 from kolibri.core.tasks.utils import get_current_job
 from kolibri.core.tasks.validation import JobValidator
-from kolibri.core.utils.urls import reverse_remote
+from kolibri.core.utils.urls import reverse_path
 from kolibri.utils import conf
 from kolibri.utils.translation import gettext as _
 from kolibri.utils.version import version_matches_range
@@ -339,14 +337,11 @@ def remoteresourceimport(
     peer_id=None,
 ):
     current_job = get_current_job()
-    metadata_url = urljoin(
-        baseurl,
-        reverse_remote(
-            baseurl, "kolibri:core:importmetadata-detail", kwargs={"pk": node_id}
-        ),
+    client = NetworkClient.build_for_address(baseurl)
+    metadata_url = reverse_path(
+        "kolibri:core:importmetadata-detail", kwargs={"pk": node_id}
     )
-    response = requests.get(metadata_url)
-    response.raise_for_status()
+    response = client.get(metadata_url)
     import_metadata = response.json()
     cancel_check = None if not current_job else current_job.check_for_cancel
     import_channel_from_data(import_metadata, cancel_check, partial=True)
@@ -641,10 +636,17 @@ class RemoteChannelDiffStatsValidator(RemoteChannelImportValidator):
     def validate(self, data):
         job_data = super(RemoteChannelDiffStatsValidator, self).validate(data)
         # get channel version metadata
+        if job_data["kwargs"]["peer_id"]:
+            client = NetworkClient.build_for_address(job_data["kwargs"]["baseurl"])
+        else:
+            client = NetworkClient("/")
         url = get_channel_lookup_url(
             baseurl=job_data["kwargs"]["baseurl"], identifier=data["channel_id"]
         )
-        resp = requests.get(url)
+        try:
+            resp = client.get(url)
+        except NetworkLocationResponseFailure as e:
+            resp = e.response
         channel_metadata = resp.json()
         job_data["extra_metadata"]["new_channel_version"] = channel_metadata[0][
             "version"

kolibri/core/content/test/test_content_app.py

@@ -7,7 +7,6 @@
 import uuid
 
 import mock
-import requests
 from django.conf import settings
 from django.core.cache import cache
 from django.test import LiveServerTestCase
@@ -28,6 +27,9 @@
 from kolibri.core.device.models import ContentCacheKey
 from kolibri.core.device.models import DevicePermissions
 from kolibri.core.device.models import DeviceSettings
+from kolibri.core.discovery.utils.network.client import NetworkClient
+from kolibri.core.discovery.utils.network.errors import NetworkLocationConnectionFailure
+from kolibri.core.discovery.utils.network.errors import NetworkLocationResponseFailure
 from kolibri.core.lessons.models import Lesson
 from kolibri.core.lessons.models import LessonAssignment
 from kolibri.core.logger.models import ContentSessionLog
@@ -1898,7 +1900,7 @@ def mock_patch_decorator(func):
     def wrapper(*args, **kwargs):
         mock_object = mock.Mock()
         mock_object.json.return_value = [{"id": 1, "name": "studio"}]
-        with mock.patch.object(requests, "get", return_value=mock_object):
+        with mock.patch.object(NetworkClient, "get", return_value=mock_object):
             return func(*args, **kwargs)
 
     return wrapper
@@ -1947,18 +1949,21 @@ def test_channel_retrieve(self):
         )
         self.assertEqual(response.data["name"], "studio")
 
-    @mock_patch_decorator
-    def test_channel_info_404(self):
-        mock_object = mock.Mock()
-        mock_object.status_code = 404
-        requests.get.return_value = mock_object
+    @mock.patch.object(
+        NetworkClient,
+        "get",
+        side_effect=NetworkLocationResponseFailure(response=mock.Mock(status_code=404)),
+    )
+    def test_channel_info_404(self, mock_get):
         response = self.client.get(
             reverse("kolibri:core:remotechannel-detail", kwargs={"pk": "abc"}),
             format="json",
         )
         self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)
 
-    @mock.patch.object(requests, "get", side_effect=requests.exceptions.ConnectionError)
+    @mock.patch.object(
+        NetworkClient, "get", side_effect=NetworkLocationConnectionFailure
+    )
     def test_channel_info_offline(self, mock_get):
         response = self.client.get(
             reverse("kolibri:core:remotechannel-detail", kwargs={"pk": "abc"}),
@@ -1967,7 +1972,9 @@ def test_channel_info_offline(self, mock_get):
         self.assertEqual(response.status_code, status.HTTP_503_SERVICE_UNAVAILABLE)
         self.assertEqual(response.json()["status"], "offline")
 
-    @mock.patch.object(requests, "get", side_effect=requests.exceptions.ConnectionError)
+    @mock.patch.object(
+        NetworkClient, "get", side_effect=NetworkLocationConnectionFailure
+    )
     def test_channel_list_offline(self, mock_get):
         response = self.client.get(
             reverse("kolibri:core:remotechannel-list"), format="json"

kolibri/core/content/test/test_file_availability.py

@@ -140,48 +140,58 @@ def setUp(self):
         process_cache.clear()
         self.location = NetworkLocation.objects.create(base_url="test")
 
-    @patch("kolibri.core.content.utils.file_availability.requests")
-    def test_set_one_file(self, requests_mock):
-        requests_mock.post.return_value.status_code = 200
-        requests_mock.post.return_value.content = "1"
+    @patch("kolibri.core.content.utils.file_availability.NetworkClient")
+    def test_set_one_file(self, networkclient_mock):
+        network_client = networkclient_mock.build_for_address.return_value
+        network_client.base_url = "test"
+        network_client.post.return_value.status_code = 200
+        network_client.post.return_value.content = "1"
         checksums = get_available_checksums_from_remote(
             test_channel_id, self.location.id
         )
         self.assertEqual(len(checksums), 1)
         self.assertTrue(local_file_qs.filter(id=list(checksums)[0]).exists())
 
-    @patch("kolibri.core.content.utils.file_availability.requests")
-    def test_set_two_files_in_channel(self, requests_mock):
-        requests_mock.post.return_value.status_code = 200
-        requests_mock.post.return_value.content = "3"
+    @patch("kolibri.core.content.utils.file_availability.NetworkClient")
+    def test_set_two_files_in_channel(self, networkclient_mock):
+        network_client = networkclient_mock.build_for_address.return_value
+        network_client.base_url = "test"
+        network_client.post.return_value.status_code = 200
+        network_client.post.return_value.content = "3"
         checksums = get_available_checksums_from_remote(
             test_channel_id, self.location.id
         )
         self.assertEqual(len(checksums), 2)
         self.assertTrue(local_file_qs.filter(id=list(checksums)[0]).exists())
         self.assertTrue(local_file_qs.filter(id=list(checksums)[1]).exists())
 
-    @patch("kolibri.core.content.utils.file_availability.requests")
-    def test_set_two_files_none_in_channel(self, requests_mock):
-        requests_mock.post.return_value.status_code = 200
-        requests_mock.post.return_value.content = "0"
+    @patch("kolibri.core.content.utils.file_availability.NetworkClient")
+    def test_set_two_files_none_in_channel(self, networkclient_mock):
+        network_client = networkclient_mock.build_for_address.return_value
+        network_client.base_url = "test"
+        network_client.post.return_value.status_code = 200
+        network_client.post.return_value.content = "0"
         checksums = get_available_checksums_from_remote(
             test_channel_id, self.location.id
         )
         self.assertEqual(checksums, set())
 
-    @patch("kolibri.core.content.utils.file_availability.requests")
-    def test_404_remote_checksum_response(self, requests_mock):
-        requests_mock.post.return_value.status_code = 404
+    @patch("kolibri.core.content.utils.file_availability.NetworkClient")
+    def test_404_remote_checksum_response(self, networkclient_mock):
+        network_client = networkclient_mock.build_for_address.return_value
+        network_client.base_url = "test"
+        network_client.post.return_value.status_code = 404
         checksums = get_available_checksums_from_remote(
             test_channel_id, self.location.id
         )
         self.assertIsNone(checksums)
 
-    @patch("kolibri.core.content.utils.file_availability.requests")
-    def test_invalid_integer_remote_checksum_response(self, requests_mock):
-        requests_mock.post.return_value.status_code = 200
-        requests_mock.post.return_value.content = "I am not a json, I am a free man!"
+    @patch("kolibri.core.content.utils.file_availability.NetworkClient")
+    def test_invalid_integer_remote_checksum_response(self, networkclient_mock):
+        network_client = networkclient_mock.build_for_address.return_value
+        network_client.base_url = "test"
+        network_client.post.return_value.status_code = 200
+        network_client.post.return_value.content = "I am not a json, I am a free man!"
         checksums = get_available_checksums_from_remote(
             test_channel_id, self.location.id
         )

kolibri/core/content/utils/file_availability.py

@@ -3,14 +3,15 @@
 import re
 from itertools import compress
 
-import requests
 from django.utils.text import compress_string
 
 from kolibri.core.content.models import LocalFile
 from kolibri.core.content.utils.channels import get_mounted_drive_by_id
 from kolibri.core.content.utils.paths import get_content_storage_dir_path
 from kolibri.core.content.utils.paths import get_file_checksums_url
 from kolibri.core.discovery.models import NetworkLocation
+from kolibri.core.discovery.utils.network.client import NetworkClient
+from kolibri.core.discovery.utils.network.errors import NetworkLocationResponseFailure
 from kolibri.core.utils.cache import process_cache
 
 checksum_regex = re.compile("^([a-f0-9]{32})$")
@@ -64,14 +65,17 @@ def get_available_checksums_from_remote(channel_id, peer_id):
             .values_list("id", flat=True)
             .distinct()
         )
-
-        response = requests.post(
-            get_file_checksums_url(channel_id, baseurl),
-            data=compress_string(
-                bytes(json.dumps(list(channel_checksums)).encode("utf-8"))
-            ),
-            headers={"content-type": "application/gzip"},
-        )
+        client = NetworkClient.build_for_address(baseurl)
+        try:
+            response = client.post(
+                get_file_checksums_url(channel_id, baseurl),
+                data=compress_string(
+                    bytes(json.dumps(list(channel_checksums)).encode("utf-8"))
+                ),
+                headers={"content-type": "application/gzip"},
+            )
+        except NetworkLocationResponseFailure as e:
+            response = e.response
 
         checksums = None