feat(beacon): Add cpu/ram usage to beacon (#65200)

This adds CPU/RAM usage to the beacon. It also adds `psutil` as a
dependency

---------

Co-authored-by: getsantry[bot] <66042841+getsantry[bot]@users.noreply.github.com>

Author: hubertdeng123

requirements-base.txt

@@ -42,6 +42,7 @@ phonenumberslite>=8.12.32
 Pillow>=10.2.0
 progressbar2>=3.41.0
 python-rapidjson>=1.4
+psutil>=5.9.2
 psycopg2-binary>=2.9.9
 PyJWT>=2.4.0
 pydantic>=1.10.9

requirements-dev-frozen.txt

@@ -127,7 +127,7 @@ progressbar2==3.41.0
 prompt-toolkit==3.0.41
 proto-plus==1.23.0
 protobuf==4.25.2
-psutil==5.9.2
+psutil==5.9.7
 psycopg2-binary==2.9.9
 pyasn1==0.4.5
 pyasn1-modules==0.2.4

requirements-dev.txt

@@ -7,7 +7,6 @@ docker>=6
 time-machine>=2.13.0
 honcho>=1.1.0
 openapi-core>=0.18.2
-psutil
 pytest>=8
 pytest-cov>=4.0.0
 pytest-django>=4.8.0

requirements-frozen.txt

@@ -86,6 +86,7 @@ progressbar2==3.41.0
 prompt-toolkit==3.0.41
 proto-plus==1.23.0
 protobuf==4.25.2
+psutil==5.9.7
 psycopg2-binary==2.9.9
 pyasn1==0.4.5
 pyasn1-modules==0.2.4

src/sentry/options/defaults.py

@@ -285,6 +285,11 @@
 
 # Beacon
 register("beacon.anonymous", type=Bool, flags=FLAG_REQUIRED)
+register(
+    "beacon.record_cpu_ram_usage",
+    type=Bool,
+    flags=FLAG_ALLOW_EMPTY | FLAG_REQUIRED,
+)
 
 # Filestore (default)
 register("filestore.backend", default="filesystem", flags=FLAG_NOSTORE)

src/sentry/tasks/beacon.py

@@ -4,6 +4,7 @@
 from hashlib import sha1
 from uuid import uuid4
 
+import psutil
 from django.conf import settings
 from django.utils import timezone
 
@@ -120,7 +121,14 @@ def send_beacon():
     # we need this to be explicitly configured and it defaults to None,
     # which is the same as False
     anonymous = options.get("beacon.anonymous") is not False
+    # getting an option sets it to the default value, so let's avoid doing that if for some reason consent prompt is somehow skipped because of this
+    send_cpu_ram_usage = (
+        options.get("beacon.record_cpu_ram_usage")
+        if options.isset("beacon.record_cpu_ram_usage")
+        else False
+    )
     event_categories_count = get_category_event_count_24h()
+    byte_in_gibibyte = 1024**3
 
     payload = {
         "install_id": install_id,
@@ -138,6 +146,14 @@ def send_beacon():
             "replays.24h": event_categories_count["replay"],
             "profiles.24h": event_categories_count["profile"],
             "monitors.24h": event_categories_count["monitor"],
+            "cpu_cores_available": psutil.cpu_count() if send_cpu_ram_usage else None,
+            "cpu_percentage_utilized": psutil.cpu_percent() if send_cpu_ram_usage else None,
+            "ram_available_gb": (
+                psutil.virtual_memory().total / byte_in_gibibyte if send_cpu_ram_usage else None
+            ),
+            "ram_percentage_utilized": (
+                psutil.virtual_memory().percent if send_cpu_ram_usage else None
+            ),
         },
         "packages": get_all_package_versions(),
         "anonymous": anonymous,

src/sentry/utils/settings.py

@@ -3,3 +3,11 @@ def is_self_hosted() -> bool:
     from django.conf import settings
 
     return settings.SENTRY_SELF_HOSTED
+
+
+def should_show_beacon_consent_prompt() -> bool:
+    from django.conf import settings
+
+    from sentry import options
+
+    return settings.SENTRY_SELF_HOSTED and not options.isset("beacon.record_cpu_ram_usage")

src/sentry/web/client_config.py

@@ -37,7 +37,7 @@
 from sentry.utils.assets import get_frontend_dist_prefix
 from sentry.utils.email import is_smtp_enabled
 from sentry.utils.http import is_using_customer_domain
-from sentry.utils.settings import is_self_hosted
+from sentry.utils.settings import is_self_hosted, should_show_beacon_consent_prompt
 from sentry.utils.support import get_support_mail
 
 
@@ -361,6 +361,8 @@ def get_context(self) -> Mapping[str, Any]:
             # Maintain isOnPremise key for backcompat (plugins?).
             "isOnPremise": is_self_hosted(),
             "isSelfHosted": is_self_hosted(),
+            "shouldShowBeaconConsentPrompt": not self.needs_upgrade
+            and should_show_beacon_consent_prompt(),
             "invitesEnabled": settings.SENTRY_ENABLE_INVITES,
             "gravatarBaseUrl": settings.SENTRY_GRAVATAR_BASE_URL,
             "termsUrl": settings.TERMS_URL,

tests/sentry/tasks/test_beacon.py

@@ -1,5 +1,6 @@
 import platform
 from datetime import timedelta
+from types import SimpleNamespace
 from unittest.mock import patch
 from uuid import uuid4
 
@@ -18,6 +19,15 @@
 
 
 @no_silo_test
+@patch("psutil.cpu_count", return_value=8)
+@patch("psutil.cpu_percent", return_value=50)
+@patch(
+    "psutil.virtual_memory",
+    return_value=SimpleNamespace(
+        total=34359738368,
+        percent=50,
+    ),
+)
 class SendBeaconTest(OutcomesSnubaTest):
     def setUp(self):
         super().setUp()
@@ -87,7 +97,15 @@ def setUp(self):
     @patch("sentry.tasks.beacon.safe_urlopen")
     @patch("sentry.tasks.beacon.safe_urlread")
     @responses.activate
-    def test_simple(self, safe_urlread, safe_urlopen, mock_get_all_package_versions):
+    def test_simple(
+        self,
+        safe_urlread,
+        safe_urlopen,
+        mock_get_all_package_versions,
+        mock_cpu_count,
+        mock_cpu_percent,
+        mock_virtual_memory,
+    ):
         self.organization
         self.project
         self.team
@@ -96,6 +114,7 @@ def test_simple(self, safe_urlread, safe_urlopen, mock_get_all_package_versions)
 
         assert options.set("system.admin-email", "[email protected]")
         assert options.set("beacon.anonymous", False)
+        assert options.set("beacon.record_cpu_ram_usage", True)
         send_beacon()
 
         install_id = options.get("sentry:install-id")
@@ -119,6 +138,10 @@ def test_simple(self, safe_urlread, safe_urlopen, mock_get_all_package_versions)
                     "replays.24h": 1,
                     "profiles.24h": 3,
                     "monitors.24h": 0,
+                    "cpu_cores_available": 8,
+                    "cpu_percentage_utilized": 50,
+                    "ram_available_gb": 32,
+                    "ram_percentage_utilized": 50,
                 },
                 "anonymous": False,
                 "admin_email": "[email protected]",
@@ -134,7 +157,75 @@ def test_simple(self, safe_urlread, safe_urlopen, mock_get_all_package_versions)
     @patch("sentry.tasks.beacon.safe_urlopen")
     @patch("sentry.tasks.beacon.safe_urlread")
     @responses.activate
-    def test_anonymous(self, safe_urlread, safe_urlopen, mock_get_all_package_versions):
+    def test_no_cpu_ram_usage(
+        self,
+        safe_urlread,
+        safe_urlopen,
+        mock_get_all_package_versions,
+        mock_cpu_count,
+        mock_cpu_percent,
+        mock_virtual_memory,
+    ):
+        self.organization
+        self.project
+        self.team
+        mock_get_all_package_versions.return_value = {"foo": "1.0"}
+        safe_urlread.return_value = json.dumps({"notices": [], "version": {"stable": "1.0.0"}})
+
+        assert options.set("system.admin-email", "[email protected]")
+        assert options.set("beacon.anonymous", False)
+        assert options.set("beacon.record_cpu_ram_usage", False)
+        send_beacon()
+
+        install_id = options.get("sentry:install-id")
+        assert install_id and len(install_id) == 40
+
+        safe_urlopen.assert_called_once_with(
+            BEACON_URL,
+            json={
+                "install_id": install_id,
+                "version": sentry.get_version(),
+                "docker": sentry.is_docker(),
+                "python_version": platform.python_version(),
+                "data": {
+                    "organizations": 2,
+                    "users": 1,
+                    "projects": 2,
+                    "teams": 2,
+                    "events.24h": 8,  # We expect the number of events to be the sum of events from two orgs. First org has 5 events while the second org has 3 events.
+                    "errors.24h": 8,
+                    "transactions.24h": 2,
+                    "replays.24h": 1,
+                    "profiles.24h": 3,
+                    "monitors.24h": 0,
+                    "cpu_cores_available": None,
+                    "cpu_percentage_utilized": None,
+                    "ram_available_gb": None,
+                    "ram_percentage_utilized": None,
+                },
+                "anonymous": False,
+                "admin_email": "[email protected]",
+                "packages": mock_get_all_package_versions.return_value,
+            },
+            timeout=5,
+        )
+        safe_urlread.assert_called_once_with(safe_urlopen.return_value)
+
+        assert options.get("sentry:latest_version") == "1.0.0"
+
+    @patch("sentry.tasks.beacon.get_all_package_versions")
+    @patch("sentry.tasks.beacon.safe_urlopen")
+    @patch("sentry.tasks.beacon.safe_urlread")
+    @responses.activate
+    def test_anonymous(
+        self,
+        safe_urlread,
+        safe_urlopen,
+        mock_get_all_package_versions,
+        mock_cpu_count,
+        mock_cpu_percent,
+        mock_virtual_memory,
+    ):
         self.organization
         self.project
         self.team
@@ -143,6 +234,7 @@ def test_anonymous(self, safe_urlread, safe_urlopen, mock_get_all_package_versio
 
         assert options.set("system.admin-email", "[email protected]")
         assert options.set("beacon.anonymous", True)
+        assert options.set("beacon.record_cpu_ram_usage", True)
         send_beacon()
 
         install_id = options.get("sentry:install-id")
@@ -166,6 +258,10 @@ def test_anonymous(self, safe_urlread, safe_urlopen, mock_get_all_package_versio
                     "replays.24h": 1,
                     "profiles.24h": 3,
                     "monitors.24h": 0,
+                    "cpu_cores_available": 8,
+                    "cpu_percentage_utilized": 50,
+                    "ram_available_gb": 32,
+                    "ram_percentage_utilized": 50,
                 },
                 "anonymous": True,
                 "packages": mock_get_all_package_versions.return_value,
@@ -180,7 +276,15 @@ def test_anonymous(self, safe_urlread, safe_urlopen, mock_get_all_package_versio
     @patch("sentry.tasks.beacon.safe_urlopen")
     @patch("sentry.tasks.beacon.safe_urlread")
     @responses.activate
-    def test_with_broadcasts(self, safe_urlread, safe_urlopen, mock_get_all_package_versions):
+    def test_with_broadcasts(
+        self,
+        safe_urlread,
+        safe_urlopen,
+        mock_get_all_package_versions,
+        mock_cpu_count,
+        mock_cpu_percent,
+        mock_virtual_memory,
+    ):
         broadcast_id = uuid4().hex
         mock_get_all_package_versions.return_value = {}
         safe_urlread.return_value = json.dumps(
@@ -236,7 +340,15 @@ def test_with_broadcasts(self, safe_urlread, safe_urlopen, mock_get_all_package_
     @patch("sentry.tasks.beacon.safe_urlopen")
     @patch("sentry.tasks.beacon.safe_urlread")
     @responses.activate
-    def test_disabled(self, safe_urlread, safe_urlopen, mock_get_all_package_versions):
+    def test_disabled(
+        self,
+        safe_urlread,
+        safe_urlopen,
+        mock_get_all_package_versions,
+        mock_cpu_count,
+        mock_cpu_percent,
+        mock_virtual_memory,
+    ):
         mock_get_all_package_versions.return_value = {"foo": "1.0"}
 
         with self.settings(SENTRY_BEACON=False):
@@ -248,7 +360,15 @@ def test_disabled(self, safe_urlread, safe_urlopen, mock_get_all_package_version
     @patch("sentry.tasks.beacon.safe_urlopen")
     @patch("sentry.tasks.beacon.safe_urlread")
     @responses.activate
-    def test_debug(self, safe_urlread, safe_urlopen, mock_get_all_package_versions):
+    def test_debug(
+        self,
+        safe_urlread,
+        safe_urlopen,
+        mock_get_all_package_versions,
+        mock_cpu_count,
+        mock_cpu_percent,
+        mock_virtual_memory,
+    ):
         mock_get_all_package_versions.return_value = {"foo": "1.0"}
 
         with self.settings(DEBUG=True):
@@ -258,7 +378,13 @@ def test_debug(self, safe_urlread, safe_urlopen, mock_get_all_package_versions):
 
     @patch("sentry.tasks.beacon.safe_urlopen")
     @responses.activate
-    def test_metrics(self, safe_urlopen):
+    def test_metrics(
+        self,
+        safe_urlopen,
+        mock_cpu_count,
+        mock_cpu_percent,
+        mock_virtual_memory,
+    ):
         metrics = [
             {
                 "description": "SentryApp",