[6.x] Fix static cache invalidation outside of a request

src/StaticCaching/Cachers/AbstractCacher.php

@@ -9,6 +9,7 @@
 use Illuminate\Http\JsonResponse;
 use Illuminate\Http\Request;
 use Illuminate\Http\Response;
+use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Cache;
 use Statamic\Console\Commands\StaticWarmJob;
 use Statamic\Facades\Site;
@@ -118,7 +119,11 @@ protected function makeHash($url)
      */
     public function getDomains()
     {
-        return collect($this->cache->get($this->normalizeKey('domains'), []));
+        // Caches written before domains were tracked per site only have the flat list.
+        return collect($this->cache->get($this->normalizeKey('domains'), []))
+            ->merge($this->getSiteDomains()->flatten())
+            ->unique()
+            ->values();
     }
 
     /**
@@ -128,13 +133,7 @@ public function getDomains()
      */
     public function cacheDomain($domain = null)
     {
-        $domains = $this->getDomains();
-
-        if (! $domains->contains($domain = $domain ?? $this->getBaseUrl())) {
-            $domains->push($domain);
-        }
-
-        $this->cache->forever($this->normalizeKey('domains'), $domains->all());
+        $this->cacheSiteDomain($domain ?? $this->getBaseUrl(), Site::current()->handle());
     }
 
     /**
@@ -155,6 +154,14 @@ public function getGeneratedAt(string $url): ?int
         return null;
     }
 
+    /**
+     * Get the domains each site has been cached under, keyed by site handle.
+     */
+    protected function getSiteDomains(): Collection
+    {
+        return collect($this->cache->get($this->normalizeKey('site-domains'), []));
+    }
+
     /**
      * Flush all the cached URLs.
      *
@@ -167,6 +174,7 @@ public function flushUrls()
         });
 
         $this->cache->forget($this->normalizeKey('domains'));
+        $this->cache->forget($this->normalizeKey('site-domains'));
     }
 
     protected function flushDedicatedStore(): void
@@ -194,7 +202,7 @@ public function cacheUrl($key, $url, $domain = null)
         $url = Str::removeLeft($url, $domain);
 
         $this->withUrlsLock(function () use ($key, $url, $domain) {
-            $this->cacheDomain($domain);
+            $this->cacheSiteDomain($domain, (Site::findByUrl($domain.$url) ?? Site::default())->handle());
 
             $urls = $this->getUrls($domain)->put($key, $url);
 
@@ -233,6 +241,17 @@ public function forgetUrls($keys, $domain = null)
         });
     }
 
+    private function cacheSiteDomain(string $domain, string $site): void
+    {
+        $siteDomains = $this->getSiteDomains();
+
+        if (in_array($domain, $domains = $siteDomains->get($site, []))) {
+            return;
+        }
+
+        $this->cache->forever($this->normalizeKey('site-domains'), $siteDomains->put($site, [...$domains, $domain])->all());
+    }
+
     private function withUrlsLock(callable $callback)
     {
         if (! ($store = $this->cache->getStore()) instanceof LockProvider) {
@@ -258,9 +277,7 @@ protected function invalidateWildcardUrl($wildcard)
 
         [$wildcard, $domain] = $this->getPathAndDomain($wildcard);
 
-        $this->getUrls($domain)->filter(function ($url) use ($wildcard) {
-            return Str::startsWith($url, $wildcard);
-        })->each(function ($url) use ($domain) {
+        $this->getWildcardMatches($wildcard, $domain)->each(function (string $url) use ($domain) {
             $this->invalidateUrl($url, $domain);
         });
     }
@@ -308,28 +325,31 @@ public function refreshUrls($urls)
      */
     public function refreshUrl($url, $domain = null)
     {
-        $this->getUrls($domain)->filter(function ($value) use ($url) {
-            return $value === $url || Str::startsWith($value, $url.'?');
-        })->each(function ($url, $key) use ($domain) {
-            // Warming an error response would just fail with the same error,
-            // so invalidate it and let the next request cache a fresh copy.
-            if ($this->hasCachedErrorResponse($key)) {
-                $this->invalidateUrl($url, $domain);
+        $domain = $domain ?: $this->getBaseUrl();
 
-                return;
-            }
+        $this->getInvalidationDomains($url, $domain)->each(function (string $target) use ($url, $domain) {
+            $this->getUrls($target)->filter(function (string $value) use ($url): bool {
+                return $value === $url || Str::startsWith($value, $url.'?');
+            })->each(function (string $url, string $key) use ($target, $domain) {
+                // Other domains were recorded from request hosts, which can be spoofed, so they're never
+                // requested. Warming an error response would just fail with the same error. In both
+                // cases, invalidate it instead and let the next request cache a fresh copy.
+                if ($target !== $domain || $this->hasCachedErrorResponse($key)) {
+                    $this->invalidateUrl($url, $target);
 
-            $url = ($domain ?: $this->getBaseUrl()).$url;
+                    return;
+                }
 
-            $url = RecacheToken::addToUrl($url);
+                $url = RecacheToken::addToUrl($domain.$url);
 
-            $request = new GuzzleRequest('GET', $url);
+                $request = new GuzzleRequest('GET', $url);
 
-            $job = config('statamic.static_caching.warm_job') ?? StaticWarmJob::class;
+                $job = config('statamic.static_caching.warm_job') ?? StaticWarmJob::class;
 
-            $job::dispatch($request, [])
-                ->onConnection(config('statamic.static_caching.warm_queue_connection') ?? config('queue.default'))
-                ->onQueue(config('statamic.static_caching.warm_queue'));
+                $job::dispatch($request, [])
+                    ->onConnection(config('statamic.static_caching.warm_queue_connection') ?? config('queue.default'))
+                    ->onQueue(config('statamic.static_caching.warm_queue'));
+            });
         });
     }
 
@@ -356,9 +376,7 @@ protected function refreshWildcardUrl($wildcard)
 
         [$wildcard, $domain] = $this->getPathAndDomain($wildcard);
 
-        $this->getUrls($domain)->filter(function ($url) use ($wildcard) {
-            return Str::startsWith($url, $wildcard);
-        })->each(function ($url) use ($domain) {
+        $this->getWildcardMatches($wildcard, $domain)->each(function (string $url) use ($domain) {
             $this->refreshUrl($url, $domain);
         });
     }
@@ -390,6 +408,45 @@ public function hasCachedPage(Request $request)
         return $this->getCachedPage($request) !== null;
     }
 
+    /**
+     * Get the domains a URL should be invalidated under.
+     *
+     * Pages are tracked under the domain they were requested from, which can differ
+     * from the domain the URL resolves to in this process. e.g. a queue worker has
+     * no request, so a relative site URL resolves against the app url instead. So
+     * the domains the URL's site has actually been cached under are included too.
+     */
+    protected function getInvalidationDomains(string $url, ?string $domain = null, ?Collection $siteDomains = null): Collection
+    {
+        $domain = $domain ?: $this->getBaseUrl();
+
+        if (is_null($site = Site::findByUrl($domain.$url))) {
+            return collect([$domain]);
+        }
+
+        $siteDomains ??= $this->getSiteDomains();
+
+        return collect([$domain, ...$siteDomains->get($site->handle(), [])])->unique()->values();
+    }
+
+    /**
+     * Get the cached URLs matching a wildcard across its invalidation domains.
+     *
+     * The domain's own matches are returned as-is, followed by any other
+     * matches the remaining domains have that aren't already included.
+     */
+    protected function getWildcardMatches(string $wildcard, ?string $domain = null, ?callable $urls = null, ?Collection $siteDomains = null): Collection
+    {
+        $urls ??= fn (string $domain): Collection => $this->getUrls($domain);
+
+        return $this->getInvalidationDomains($wildcard, $domain, $siteDomains)->reduce(function (Collection $matches, string $domain, int $index) use ($urls, $wildcard): Collection {
+            return $matches->concat(collect($urls($domain))
+                ->filter(fn (string $url): bool => Str::startsWith($url, $wildcard))
+                ->reject(fn (string $url): bool => $index > 0 && $matches->contains($url))
+                ->values());
+        }, collect());
+    }
+
     protected function getPathAndDomain($url)
     {
         $parsed = parse_url($url);

src/StaticCaching/Cachers/ApplicationCacher.php

@@ -4,6 +4,7 @@
 
 use Illuminate\Http\Request;
 use Illuminate\Routing\Events\ResponsePrepared;
+use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Event;
 use Statamic\Events\UrlInvalidated;
 use Statamic\StaticCaching\Page;
@@ -165,14 +166,16 @@ public function invalidateUrl($url, $domain = null)
             [$url, $domain] = $this->getPathAndDomain($url);
         }
 
-        $this
-            ->getUrls($domain)
-            ->filter(fn ($value) => $value === $url || str_starts_with($value, $url.'?'))
-            ->each(function ($value, $key) use ($domain) {
-                $this->cache->forget($this->normalizeKey('responses:'.$key));
-                $this->cache->forget($this->normalizeKey('generated-at:'.$key));
-                $this->forgetUrl($key, $domain);
-            });
+        $this->getInvalidationDomains($url, $domain)->each(function (string $domain) use ($url) {
+            $this
+                ->getUrls($domain)
+                ->filter(fn (string $value): bool => $value === $url || str_starts_with($value, $url.'?'))
+                ->each(function (string $value, string $key) use ($domain) {
+                    $this->cache->forget($this->normalizeKey('responses:'.$key));
+                    $this->cache->forget($this->normalizeKey('generated-at:'.$key));
+                    $this->forgetUrl($key, $domain);
+                });
+        });
 
         UrlInvalidated::dispatch($url, $domain);
     }
@@ -199,9 +202,18 @@ public function invalidateUrls($urls)
         $removed = [];
         $events = [];
 
-        $invalidate = function ($path, $domain) use (&$indexes, &$removed, &$events) {
-            foreach ($indexes[$domain]['paths'][$path] ?? [] as $key) {
-                $removed[$domain][$key] = true;
+        // Read once, as every URL and wildcard match looks up the domains to invalidate it under.
+        $siteDomains = $this->getSiteDomains();
+
+        $index = function (string $domain) use (&$indexes): array {
+            return $indexes[$domain] ??= $this->getUrlIndex($domain);
+        };
+
+        $invalidate = function (string $path, string $domain) use ($index, $siteDomains, &$removed, &$events): void {
+            foreach ($this->getInvalidationDomains($path, $domain, $siteDomains) as $target) {
+                foreach ($index($target)['paths'][$path] ?? [] as $key) {
+                    $removed[$target][$key] = true;
+                }
             }
 
             $events[] = [$path, $domain];
@@ -212,19 +224,17 @@ public function invalidateUrls($urls)
 
             [$path, $domain] = $this->getPathAndDomain($wildcard ? substr($url, 0, -1) : $url);
 
-            $indexes[$domain] ??= $this->getUrlIndex($domain);
-
             if (! $wildcard) {
                 $invalidate($path, $domain);
 
                 continue;
             }
 
-            // Match against the index as it stands before this wildcard, like
+            // Match against the indexes as they stand before this wildcard, like
             // invalidateWildcardUrl() does, then invalidate each match in turn.
-            $matches = collect($indexes[$domain]['urls'])
-                ->reject(fn ($value, $key) => isset($removed[$domain][$key]))
-                ->filter(fn ($value) => Str::startsWith($value, $path));
+            $matches = $this->getWildcardMatches($path, $domain, function (string $target) use ($index, &$removed): Collection {
+                return collect($index($target)['urls'])->reject(fn (string $value, string $key): bool => isset($removed[$target][$key]));
+            }, $siteDomains);
 
             foreach ($matches as $match) {
                 $invalidate($match, $domain);

src/StaticCaching/Cachers/FileCacher.php

@@ -147,13 +147,15 @@ public function invalidateUrl($url, $domain = null)
     {
         $site = optional(Site::findByUrl($domain.$url))->handle();
 
-        $this
-            ->getUrls($domain)
-            ->filter(fn ($value) => $value === $url || str_starts_with($value, $url.'?'))
-            ->each(function ($value, $key) use ($site, $domain) {
-                $this->writer->delete($this->getFilePath($domain.$value, $site));
-                $this->forgetUrl($key, $domain);
-            });
+        $this->getInvalidationDomains($url, $domain)->each(function (string $domain) use ($site, $url) {
+            $this
+                ->getUrls($domain)
+                ->filter(fn (string $value): bool => $value === $url || str_starts_with($value, $url.'?'))
+                ->each(function (string $value, string $key) use ($site, $domain) {
+                    $this->writer->delete($this->getFilePath($domain.$value, $site));
+                    $this->forgetUrl($key, $domain);
+                });
+        });
 
         $filePath = $this->getFilePath($domain.$url, $site);