don't rate limit public Fastly IPs (#54625)

Author: Ebonsignori

src/shielding/lib/fastly-ips.ts

@@ -0,0 +1,81 @@
+// Logic to get and store the current list of public Fastly IPs from the Fastly API: https://www.fastly.com/documentation/reference/api/utils/public-ip-list/
+
+// Default returned from ➜ curl "https://api.fastly.com/public-ip-list"
+export const DEFAULT_FASTLY_IPS: string[] = [
+  '23.235.32.0/20',
+  '43.249.72.0/22',
+  '103.244.50.0/24',
+  '103.245.222.0/23',
+  '103.245.224.0/24',
+  '104.156.80.0/20',
+  '140.248.64.0/18',
+  '140.248.128.0/17',
+  '146.75.0.0/17',
+  '151.101.0.0/16',
+  '157.52.64.0/18',
+  '167.82.0.0/17',
+  '167.82.128.0/20',
+  '167.82.160.0/20',
+  '167.82.224.0/20',
+  '172.111.64.0/18',
+  '185.31.16.0/22',
+  '199.27.72.0/21',
+  '199.232.0.0/16',
+]
+
+let ipCache: string[] = []
+
+export async function getPublicFastlyIPs(): Promise<string[]> {
+  // Don't fetch the list in dev & testing, just use the defaults
+  if (process.env.NODE_ENV !== 'production') {
+    ipCache = DEFAULT_FASTLY_IPS
+  }
+
+  if (ipCache.length) {
+    return ipCache
+  }
+
+  const endpoint = 'https://api.fastly.com/public-ip-list'
+  let ips: string[] = []
+  let attempt = 0
+
+  while (attempt < 3) {
+    try {
+      const response = await fetch(endpoint)
+      if (!response.ok) {
+        throw new Error(`Failed to fetch: ${response.status}`)
+      }
+      const data = await response.json()
+      if (data && Array.isArray(data.addresses)) {
+        ips = data.addresses
+        break
+      } else {
+        throw new Error('Invalid response structure')
+      }
+    } catch (error: any) {
+      console.error(
+        `Failed to fetch Fastly IPs: ${error.message}. Retrying ${3 - attempt} more times`,
+      )
+      attempt++
+      if (attempt >= 3) {
+        ips = DEFAULT_FASTLY_IPS
+      }
+    }
+  }
+
+  ipCache = ips
+  return ips
+}
+
+// The IPs we check in the rate-limiter are in the form `X.X.X.X`
+// But the IPs returned from the Fastly API are in the form `X.X.X.X/Y`
+// For an IP in the rate-limiter, we want `X.X.X.*` to match `X.X.X.X/Y`
+export async function isFastlyIP(ip: string): Promise<boolean> {
+  // If IPs aren't initialized, fetch them
+  if (!ipCache.length) {
+    await getPublicFastlyIPs()
+  }
+  const parts = ip.split('.')
+  const prefix = parts.slice(0, 3).join('.')
+  return ipCache.some((fastlyIP) => fastlyIP.startsWith(prefix))
+}

src/shielding/middleware/rate-limit.ts

@@ -4,6 +4,7 @@ import rateLimit from 'express-rate-limit'
 
 import statsd from '@/observability/lib/statsd.js'
 import { noCacheControl } from '@/frame/middleware/cache-control.js'
+import { isFastlyIP } from '@/shielding/lib/fastly-ips'
 
 const EXPIRES_IN_AS_SECONDS = 60
 
@@ -35,8 +36,11 @@ export function createRateLimiter(max = MAX, isAPILimiter = false) {
       return getClientIPFromReq(req)
     },
 
-    skip: (req) => {
+    skip: async (req) => {
       const ip = getClientIPFromReq(req)
+      if (await isFastlyIP(ip)) {
+        return true
+      }
       // IP is empty when we are in a non-production (not behind Fastly) environment
       // In these environments, we don't want to rate limit (including tests)
       // However, if you want to test rate limiting locally, you can manually set

src/shielding/tests/shielding.ts

@@ -2,6 +2,7 @@ import { describe, expect, test } from 'vitest'
 
 import { SURROGATE_ENUMS } from '@/frame/middleware/set-fastly-surrogate-key.js'
 import { get } from '@/tests/helpers/e2etest.js'
+import { DEFAULT_FASTLY_IPS } from '@/shielding/lib/fastly-ips'
 
 describe('honeypotting', () => {
   test('any GET with survey-vote and survey-token query strings is 400', async () => {
@@ -136,6 +137,51 @@ describe('rate limiting', () => {
     expect(res.headers['ratelimit-limit']).toBeUndefined()
     expect(res.headers['ratelimit-remaining']).toBeUndefined()
   })
+
+  test('/api/cookies only allows 1 request per minute', async () => {
+    // Cookies only allows 1 request per minute
+    const res1 = await get('/api/cookies', {
+      headers: {
+        'fastly-client-ip': 'abc123',
+      },
+    })
+    expect(res1.statusCode).toBe(200)
+    expect(res1.headers['ratelimit-limit']).toBe('1')
+    expect(res1.headers['ratelimit-remaining']).toBe('0')
+
+    // A second request should be rate limited
+    const res2 = await get('/api/cookies', {
+      headers: {
+        'fastly-client-ip': 'abc123',
+      },
+    })
+    expect(res2.statusCode).toBe(429)
+    expect(res2.headers['ratelimit-limit']).toBe('1')
+    expect(res2.headers['ratelimit-remaining']).toBe('0')
+  })
+
+  test('Fastly IPs are not rate limited', async () => {
+    // Fastly IPs are in the form `X.X.X.X/Y`
+    // Rate limited IPs are in the form `X.X.X.X`
+    // Where the last X could be any 2-3 digit number
+    const mockFastlyIP =
+      DEFAULT_FASTLY_IPS[0].split('.').slice(0, 3).join('.') + `.${Math.floor(Math.random() * 100)}`
+    // Cookies only allows 1 request per minute
+    const res1 = await get('/api/cookies', {
+      headers: {
+        'fastly-client-ip': mockFastlyIP,
+      },
+    })
+    expect(res1.statusCode).toBe(200)
+
+    // A second request shouldn't be rate limited because it's from a Fastly IP
+    const res2 = await get('/api/cookies', {
+      headers: {
+        'fastly-client-ip': mockFastlyIP,
+      },
+    })
+    expect(res2.statusCode).toBe(200)
+  })
 })
 
 describe('404 pages and their content-type', () => {