Sudden HTTP error: 404 page not found

[abratashov]

After four months, my production server started failing. When I follow the web link, I see 404 page not found.

Logs shows:

$ kamal traefik logs -d prod
Traefik Host: 1.2.3.4
2024-10-08T15:26:40.902889581Z time="2024-10-08T15:26:40Z" level=debug msg="mime: no media type" middlewareName=traefik-compress@docker middlewareType=Compress
2024-10-08T15:26:40.906863811Z time="2024-10-08T15:26:40Z" level=debug msg="mime: no media type" middlewareName=traefik-compress@docker middlewareType=Compress
2024-10-08T15:26:40.911196263Z time="2024-10-08T15:26:40Z" level=debug msg="mime: no media type" middlewareName=traefik-compress@docker middlewareType=Compress
2024-10-08T15:26:41.303825662Z time="2024-10-08T15:26:41Z" level=debug msg="mime: no media type" middlewareType=Compress middlewareName=traefik-compress@docker
2024-10-08T15:26:50.520949690Z time="2024-10-08T15:26:50Z" level=debug msg="mime: no media type" middlewareName=traefik-compress@docker middlewareType=Compress
2024-10-08T15:26:53.493290946Z time="2024-10-08T15:26:53Z" level=debug msg="mime: no media type" middlewareType=Compress middlewareName=traefik-compress@docker
2024-10-08T15:26:54.035447436Z time="2024-10-08T15:26:54Z" level=debug msg="mime: no media type" middlewareType=Compress middlewareName=traefik-compress@docker
2024-10-08T15:27:12.871873533Z time="2024-10-08T15:27:12Z" level=debug msg="mime: no media type" middlewareName=traefik-compress@docker middlewareType=Compress
2024-10-08T15:27:20.300228483Z time="2024-10-08T15:27:20Z" level=debug msg="Provider event received {Status:die ID:25cd130d1c61da1c04c1c3d4a77563e7be1e0a15dda91cb036b0fa637e8a05ab From:registry.gitlab.com/abratashov/my-site/my-site-prod:40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e Type:container Action:die Actor:{ID:25cd130d1c61da1c04c1c3d4a77563e7be1e0a15dda91cb036b0fa637e8a05ab Attributes:map[destination: execDuration:1129656 exitCode:1 image:registry.gitlab.com/abratashov/my-site/my-site-prod:40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e name:my-site-prod-web-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e role:web service:my-site-prod traefik.http.middlewares.my-site-prod-web-retry.retry.attempts:5 traefik.http.middlewares.my-site-prod-web-retry.retry.initialinterval:500ms traefik.http.middlewares.traefik-compress.compress:true traefik.http.routers.my-site-prod-web.middlewares:my-site-prod-web-retry@docker traefik.http.routers.my-site-prod-web.priority:2 traefik.http.routers.my-site-prod-web.rule:PathPrefix(`/`) traefik.http.routers.my_site.rule:Host(`newacropolis.org.ua`) traefik.http.routers.my_site_secure.entrypoints:websecure traefik.http.routers.my_site_secure.middlewares:traefik-compress traefik.http.routers.my_site_secure.rule:Host(`newacropolis.org.ua`) traefik.http.routers.my_site_secure.tls:true traefik.http.routers.my_site_secure.tls.certresolver:letsencrypt traefik.http.services.my-site-prod-web.loadbalancer.server.scheme:http]} Scope:local Time:1728401240 TimeNano:1728401240297881937}" providerName=docker
2024-10-08T15:27:20.797624170Z time="2024-10-08T15:27:20Z" level=debug msg="Filtering unhealthy or starting container" providerName=docker container=my-site-prod-web-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e-25cd130d1c61da1c04c1c3d4a77563e7be1e0a15dda91cb036b0fa637e8a05ab
2024-10-08T15:27:20.797683167Z time="2024-10-08T15:27:20Z" level=error msg="service \"my-site-prod-db-backup\" error: port is missing" providerName=docker container=my-site-prod-db-backup-14b3adb74d68dfcf4b59e11ee10549de12dabc0b560490a4b98a1c555049e0d9
2024-10-08T15:27:20.797693753Z time="2024-10-08T15:27:20Z" level=debug msg="Provider event received {Status:start ID:25cd130d1c61da1c04c1c3d4a77563e7be1e0a15dda91cb036b0fa637e8a05ab From:registry.gitlab.com/abratashov/my-site/my-site-prod:40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e Type:container Action:start Actor:{ID:25cd130d1c61da1c04c1c3d4a77563e7be1e0a15dda91cb036b0fa637e8a05ab Attributes:map[destination: image:registry.gitlab.com/abratashov/my-site/my-site-prod:40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e name:my-site-prod-web-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e role:web service:my-site-prod traefik.http.middlewares.my-site-prod-web-retry.retry.attempts:5 traefik.http.middlewares.my-site-prod-web-retry.retry.initialinterval:500ms traefik.http.middlewares.traefik-compress.compress:true traefik.http.routers.my-site-prod-web.middlewares:my-site-prod-web-retry@docker traefik.http.routers.my-site-prod-web.priority:2 traefik.http.routers.my-site-prod-web.rule:PathPrefix(`/`) traefik.http.routers.my_site.rule:Host(`newacropolis.org.ua`) traefik.http.routers.my_site_secure.entrypoints:websecure traefik.http.routers.my_site_secure.middlewares:traefik-compress traefik.http.routers.my_site_secure.rule:Host(`newacropolis.org.ua`) traefik.http.routers.my_site_secure.tls:true traefik.http.routers.my_site_secure.tls.certresolver:letsencrypt traefik.http.services.my-site-prod-web.loadbalancer.server.scheme:http]} Scope:local Time:1728401240 TimeNano:1728401240777542676}" providerName=docker
2024-10-08T15:27:20.801437013Z time="2024-10-08T15:27:20Z" level=debug msg="Configuration received: {\"http\":{\"routers\":{\"catchall\":{\"entryPoints\":[\"http\"],\"service\":\"unavailable\",\"rule\":\"PathPrefix(`/`)\",\"priority\":1},\"my-site-prod-db\":{\"service\":\"my-site-prod-db\",\"rule\":\"Host(`my-site-prod-db`)\"},\"my-site-prod-job-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e\":{\"service\":\"my-site-prod-job-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e\",\"rule\":\"Host(`my-site-prod-job-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e`)\"},\"my-site-prod-job-prod-latest-prod\":{\"service\":\"my-site-prod-job-prod-latest-prod\",\"rule\":\"Host(`my-site-prod-job-prod-latest-prod`)\"},\"my-site-prod-redis\":{\"service\":\"my-site-prod-redis\",\"rule\":\"Host(`my-site-prod-redis`)\"},\"my-site-prod-web-prod\":{\"middlewares\":[\"my-site-prod-web-prod-retry@docker\"],\"service\":\"my-site-prod-web-prod\",\"rule\":\"PathPrefix(`/`)\",\"priority\":2},\"my_site\":{\"service\":\"my-site-prod-web-prod\",\"rule\":\"Host(`newacropolis.org.ua`)\"},\"my_site_secure\":{\"entryPoints\":[\"websecure\"],\"middlewares\":[\"traefik-compress\"],\"service\":\"my-site-prod-web-prod\",\"rule\":\"Host(`newacropolis.org.ua`)\",\"tls\":{\"certResolver\":\"letsencrypt\"}}},\"services\":{\"my-site-prod-db\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://172.17.0.3:5432\"}],\"passHostHeader\":true}},\"my-site-prod-job-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://172.17.0.6:3000\"}],\"passHostHeader\":true}},\"my-site-prod-job-prod-latest-prod\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://172.17.0.9:3000\"}],\"passHostHeader\":true}},\"my-site-prod-redis\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://172.17.0.4:6379\"}],\"passHostHeader\":true}},\"my-site-prod-web-prod\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://172.17.0.5:3000\"}],\"passHostHeader\":true}},\"unavailable\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://172.17.0.2:0\"}],\"passHostHeader\":true}}},\"middlewares\":{\"my-site-prod-web-prod-retry\":{\"retry\":{\"attempts\":5,\"initialInterval\":\"500ms\"}},\"traefik-compress\":{\"compress\":{}}}},\"tcp\":{},\"udp\":{}}" providerName=docker
2024-10-08T15:27:20.801507723Z time="2024-10-08T15:27:20Z" level=debug msg="Skipping unchanged configuration." providerName=docker
2024-10-08T15:27:20.811774477Z time="2024-10-08T15:27:20Z" level=debug msg="Filtering unhealthy or starting container" providerName=docker container=my-site-prod-web-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e-25cd130d1c61da1c04c1c3d4a77563e7be1e0a15dda91cb036b0fa637e8a05ab
2024-10-08T15:27:20.811837534Z time="2024-10-08T15:27:20Z" level=error msg="service \"my-site-prod-db-backup\" error: port is missing" container=my-site-prod-db-backup-14b3adb74d68dfcf4b59e11ee10549de12dabc0b560490a4b98a1c555049e0d9 providerName=docker
2024-10-08T15:27:22.799034142Z time="2024-10-08T15:27:22Z" level=debug msg="Configuration received: {\"http\":{\"routers\":{\"catchall\":{\"entryPoints\":[\"http\"],\"service\":\"unavailable\",\"rule\":\"PathPrefix(`/`)\",\"priority\":1},\"my-site-prod-db\":{\"service\":\"my-site-prod-db\",\"rule\":\"Host(`my-site-prod-db`)\"},\"my-site-prod-job-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e\":{\"service\":\"my-site-prod-job-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e\",\"rule\":\"Host(`my-site-prod-job-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e`)\"},\"my-site-prod-job-prod-latest-prod\":{\"service\":\"my-site-prod-job-prod-latest-prod\",\"rule\":\"Host(`my-site-prod-job-prod-latest-prod`)\"},\"my-site-prod-redis\":{\"service\":\"my-site-prod-redis\",\"rule\":\"Host(`my-site-prod-redis`)\"},\"my-site-prod-web-prod\":{\"middlewares\":[\"my-site-prod-web-prod-retry@docker\"],\"service\":\"my-site-prod-web-prod\",\"rule\":\"PathPrefix(`/`)\",\"priority\":2},\"my_site\":{\"service\":\"my-site-prod-web-prod\",\"rule\":\"Host(`newacropolis.org.ua`)\"},\"my_site_secure\":{\"entryPoints\":[\"websecure\"],\"middlewares\":[\"traefik-compress\"],\"service\":\"my-site-prod-web-prod\",\"rule\":\"Host(`newacropolis.org.ua`)\",\"tls\":{\"certResolver\":\"letsencrypt\"}}},\"services\":{\"my-site-prod-db\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://172.17.0.3:5432\"}],\"passHostHeader\":true}},\"my-site-prod-job-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://172.17.0.6:3000\"}],\"passHostHeader\":true}},\"my-site-prod-job-prod-latest-prod\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://172.17.0.9:3000\"}],\"passHostHeader\":true}},\"my-site-prod-redis\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://172.17.0.4:6379\"}],\"passHostHeader\":true}},\"my-site-prod-web-prod\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://172.17.0.5:3000\"}],\"passHostHeader\":true}},\"unavailable\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://172.17.0.2:0\"}],\"passHostHeader\":true}}},\"middlewares\":{\"my-site-prod-web-prod-retry\":{\"retry\":{\"attempts\":5,\"initialInterval\":\"500ms\"}},\"traefik-compress\":{\"compress\":{}}}},\"tcp\":{},\"udp\":{}}" providerName=docker
2024-10-08T15:27:22.799219196Z time="2024-10-08T15:27:22Z" level=debug msg="Skipping unchanged configuration." providerName=docker
2024-10-08T15:27:46.329386076Z time="2024-10-08T15:27:46Z" level=debug msg="mime: no media type" middlewareName=traefik-compress@docker middlewareType=Compress
2024-10-08T15:28:04.854627771Z time="2024-10-08T15:28:04Z" level=debug msg="mime: no media type" middlewareName=traefik-compress@docker middlewareType=Compress
2024-10-08T15:28:08.978994091Z time="2024-10-08T15:28:08Z" level=debug msg="mime: no media type" middlewareName=traefik-compress@docker middlewareType=Compress
2024-10-08T15:28:21.071487626Z time="2024-10-08T15:28:21Z" level=debug msg="Provider event received {Status:health_status: healthy ID:25cd130d1c61da1c04c1c3d4a77563e7be1e0a15dda91cb036b0fa637e8a05ab From:registry.gitlab.com/abratashov/my-site/my-site-prod:40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e Type:container Action:health_status: healthy Actor:{ID:25cd130d1c61da1c04c1c3d4a77563e7be1e0a15dda91cb036b0fa637e8a05ab Attributes:map[destination: image:registry.gitlab.com/abratashov/my-site/my-site-prod:40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e name:my-site-prod-web-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e role:web service:my-site-prod traefik.http.middlewares.my-site-prod-web-retry.retry.attempts:5 traefik.http.middlewares.my-site-prod-web-retry.retry.initialinterval:500ms traefik.http.middlewares.traefik-compress.compress:true traefik.http.routers.my-site-prod-web.middlewares:my-site-prod-web-retry@docker traefik.http.routers.my-site-prod-web.priority:2 traefik.http.routers.my-site-prod-web.rule:PathPrefix(`/`) traefik.http.routers.my_site.rule:Host(`newacropolis.org.ua`) traefik.http.routers.my_site_secure.entrypoints:websecure traefik.http.routers.my_site_secure.middlewares:traefik-compress traefik.http.routers.my_site_secure.rule:Host(`newacropolis.org.ua`) traefik.http.routers.my_site_secure.tls:true traefik.http.routers.my_site_secure.tls.certresolver:letsencrypt traefik.http.services.my-site-prod-web.loadbalancer.server.scheme:http]} Scope:local Time:1728401301 TimeNano:1728401301069729378}" providerName=docker
2024-10-08T15:28:21.094491395Z time="2024-10-08T15:28:21Z" level=error msg="service \"my-site-prod-db-backup\" error: port is missing" providerName=docker container=my-site-prod-db-backup-14b3adb74d68dfcf4b59e11ee10549de12dabc0b560490a4b98a1c555049e0d9
2024-10-08T15:28:21.095908793Z time="2024-10-08T15:28:21Z" level=error msg="Router defined multiple times with different configurations in [my-site-prod-web-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e-25cd130d1c61da1c04c1c3d4a77563e7be1e0a15dda91cb036b0fa637e8a05ab my-site-prod-web-prod-latest-prod-38e2872ecc6bc332198a603eb3cf6323522ecaeba12eb048c7ff7cbeacf09c9d]" routerName=my_site providerName=docker
2024-10-08T15:28:21.096072579Z time="2024-10-08T15:28:21Z" level=error msg="Router defined multiple times with different configurations in [my-site-prod-web-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e-25cd130d1c61da1c04c1c3d4a77563e7be1e0a15dda91cb036b0fa637e8a05ab my-site-prod-web-prod-latest-prod-38e2872ecc6bc332198a603eb3cf6323522ecaeba12eb048c7ff7cbeacf09c9d]" routerName=my_site_secure providerName=docker
2024-10-08T15:28:21.096838941Z time="2024-10-08T15:28:21Z" level=debug msg="Configuration received: {\"http\":{\"routers\":{\"catchall\":{\"entryPoints\":[\"http\"],\"service\":\"unavailable\",\"rule\":\"PathPrefix(`/`)\",\"priority\":1},\"my-site-prod-db\":{\"service\":\"my-site-prod-db\",\"rule\":\"Host(`my-site-prod-db`)\"},\"my-site-prod-job-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e\":{\"service\":\"my-site-prod-job-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e\",\"rule\":\"Host(`my-site-prod-job-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e`)\"},\"my-site-prod-job-prod-latest-prod\":{\"service\":\"my-site-prod-job-prod-latest-prod\",\"rule\":\"Host(`my-site-prod-job-prod-latest-prod`)\"},\"my-site-prod-redis\":{\"service\":\"my-site-prod-redis\",\"rule\":\"Host(`my-site-prod-redis`)\"},\"my-site-prod-web\":{\"middlewares\":[\"my-site-prod-web-retry@docker\"],\"service\":\"my-site-prod-web\",\"rule\":\"PathPrefix(`/`)\",\"priority\":2},\"my-site-prod-web-prod\":{\"middlewares\":[\"my-site-prod-web-prod-retry@docker\"],\"service\":\"my-site-prod-web-prod\",\"rule\":\"PathPrefix(`/`)\",\"priority\":2}},\"services\":{\"my-site-prod-db\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://172.17.0.3:5432\"}],\"passHostHeader\":true}},\"my-site-prod-job-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://172.17.0.6:3000\"}],\"passHostHeader\":true}},\"my-site-prod-job-prod-latest-prod\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://172.17.0.9:3000\"}],\"passHostHeader\":true}},\"my-site-prod-redis\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://172.17.0.4:6379\"}],\"passHostHeader\":true}},\"my-site-prod-web\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://172.17.0.8:3000\"}],\"passHostHeader\":true}},\"my-site-prod-web-prod\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://172.17.0.5:3000\"}],\"passHostHeader\":true}},\"unavailable\":{\"loadBalancer\":{\"servers\":[{\"url\":\"http://172.17.0.2:0\"}],\"passHostHeader\":true}}},\"middlewares\":{\"my-site-prod-web-prod-retry\":{\"retry\":{\"attempts\":5,\"initialInterval\":\"500ms\"}},\"my-site-prod-web-retry\":{\"retry\":{\"attempts\":5,\"initialInterval\":\"500ms\"}},\"traefik-compress\":{\"compress\":{}}}},\"tcp\":{},\"udp\":{}}" providerName=docker
2024-10-08T15:28:21.097558473Z time="2024-10-08T15:28:21Z" level=debug msg="No entryPoint defined for this router, using the default one(s) instead: [web websecure]" routerName=my-site-prod-web
2024-10-08T15:28:21.097749393Z time="2024-10-08T15:28:21Z" level=debug msg="No entryPoint defined for this router, using the default one(s) instead: [web websecure]" routerName=my-site-prod-db
2024-10-08T15:28:21.097968613Z time="2024-10-08T15:28:21Z" level=debug msg="No entryPoint defined for this router, using the default one(s) instead: [web websecure]" routerName=my-site-prod-job-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e
2024-10-08T15:28:21.098146470Z time="2024-10-08T15:28:21Z" level=debug msg="No entryPoint defined for this router, using the default one(s) instead: [web websecure]" routerName=my-site-prod-web-prod
2024-10-08T15:28:21.098373123Z time="2024-10-08T15:28:21Z" level=debug msg="No entryPoint defined for this router, using the default one(s) instead: [web websecure]" routerName=my-site-prod-job-prod-latest-prod
2024-10-08T15:28:21.098545065Z time="2024-10-08T15:28:21Z" level=debug msg="No entryPoint defined for this router, using the default one(s) instead: [web websecure]" routerName=my-site-prod-redis
2024-10-08T15:28:21.101919066Z time="2024-10-08T15:28:21Z" level=debug msg="Adding certificate for domain(s) demo.newacropolis.org.ua"
2024-10-08T15:28:21.104084059Z time="2024-10-08T15:28:21Z" level=debug msg="Adding certificate for domain(s) newacropolis.org.ua"
2024-10-08T15:28:21.695244803Z time="2024-10-08T15:28:21Z" level=debug msg="No default certificate, fallback to the internal generated certificate" tlsStoreName=default
2024-10-08T15:28:21.697119322Z time="2024-10-08T15:28:21Z" level=error msg="entryPoint \"http\" doesn't exist" routerName=catchall@docker entryPointName=http
2024-10-08T15:28:21.697136296Z time="2024-10-08T15:28:21Z" level=error msg="no valid entryPoint for this router" routerName=catchall@docker
2024-10-08T15:28:21.698986697Z time="2024-10-08T15:28:21Z" level=debug msg="Creating middleware" serviceName=my-site-prod-job-prod-latest-prod middlewareName=pipelining middlewareType=Pipelining entryPointName=web routerName=my-site-prod-job-prod-latest-prod@docker
2024-10-08T15:28:21.699002857Z time="2024-10-08T15:28:21Z" level=debug msg="Creating load-balancer" entryPointName=web routerName=my-site-prod-job-prod-latest-prod@docker serviceName=my-site-prod-job-prod-latest-prod
2024-10-08T15:28:21.699085122Z time="2024-10-08T15:28:21Z" level=debug msg="Creating server 0 http://172.17.0.9:3000" routerName=my-site-prod-job-prod-latest-prod@docker serviceName=my-site-prod-job-prod-latest-prod serverName=0 entryPointName=web
2024-10-08T15:28:21.699096760Z time="2024-10-08T15:28:21Z" level=debug msg="child http://172.17.0.9:3000 now UP"
2024-10-08T15:28:21.699103488Z time="2024-10-08T15:28:21Z" level=debug msg="Propagating new UP status"
2024-10-08T15:28:21.699185796Z time="2024-10-08T15:28:21Z" level=debug msg="Added outgoing tracing middleware my-site-prod-job-prod-latest-prod" routerName=my-site-prod-job-prod-latest-prod@docker middlewareName=tracing middlewareType=TracingForwarder entryPointName=web
2024-10-08T15:28:21.700492229Z time="2024-10-08T15:28:21Z" level=debug msg="Creating middleware" middlewareType=Pipelining serviceName=my-site-prod-redis entryPointName=web routerName=my-site-prod-redis@docker middlewareName=pipelining
2024-10-08T15:28:21.700505584Z time="2024-10-08T15:28:21Z" level=debug msg="Creating load-balancer" entryPointName=web routerName=my-site-prod-redis@docker serviceName=my-site-prod-redis
2024-10-08T15:28:21.700575150Z time="2024-10-08T15:28:21Z" level=debug msg="Creating server 0 http://172.17.0.4:6379" serverName=0 entryPointName=web routerName=my-site-prod-redis@docker serviceName=my-site-prod-redis
2024-10-08T15:28:21.700586414Z time="2024-10-08T15:28:21Z" level=debug msg="child http://172.17.0.4:6379 now UP"
2024-10-08T15:28:21.700592987Z time="2024-10-08T15:28:21Z" level=debug msg="Propagating new UP status"
2024-10-08T15:28:21.700599273Z time="2024-10-08T15:28:21Z" level=debug msg="Added outgoing tracing middleware my-site-prod-redis" middlewareType=TracingForwarder middlewareName=tracing entryPointName=web routerName=my-site-prod-redis@docker
2024-10-08T15:28:21.700661026Z time="2024-10-08T15:28:21Z" level=debug msg="Added outgoing tracing middleware acme-http@internal" middlewareType=TracingForwarder entryPointName=web routerName=acme-http@internal middlewareName=tracing
2024-10-08T15:28:21.700945241Z time="2024-10-08T15:28:21Z" level=debug msg="Creating middleware" serviceName=my-site-prod-web middlewareName=pipelining middlewareType=Pipelining entryPointName=web routerName=my-site-prod-web@docker
2024-10-08T15:28:21.700962418Z time="2024-10-08T15:28:21Z" level=debug msg="Creating load-balancer" entryPointName=web routerName=my-site-prod-web@docker serviceName=my-site-prod-web
2024-10-08T15:28:21.701039171Z time="2024-10-08T15:28:21Z" level=debug msg="Creating server 0 http://172.17.0.8:3000" entryPointName=web routerName=my-site-prod-web@docker serviceName=my-site-prod-web serverName=0
2024-10-08T15:28:21.701050432Z time="2024-10-08T15:28:21Z" level=debug msg="child http://172.17.0.8:3000 now UP"
2024-10-08T15:28:21.701056892Z time="2024-10-08T15:28:21Z" level=debug msg="Propagating new UP status"
2024-10-08T15:28:21.701062999Z time="2024-10-08T15:28:21Z" level=debug msg="Added outgoing tracing middleware my-site-prod-web" middlewareType=TracingForwarder routerName=my-site-prod-web@docker entryPointName=web middlewareName=tracing
2024-10-08T15:28:21.701069542Z time="2024-10-08T15:28:21Z" level=debug msg="Creating middleware" middlewareType=Retry entryPointName=web routerName=my-site-prod-web@docker middlewareName=my-site-prod-web-retry@docker
2024-10-08T15:28:21.701143417Z time="2024-10-08T15:28:21Z" level=debug msg="Adding tracing to middleware" entryPointName=web middlewareName=my-site-prod-web-retry@docker routerName=my-site-prod-web@docker
2024-10-08T15:28:21.701155055Z time="2024-10-08T15:28:21Z" level=debug msg="Creating middleware" middlewareName=pipelining entryPointName=web routerName=my-site-prod-db@docker serviceName=my-site-prod-db middlewareType=Pipelining
2024-10-08T15:28:21.701161805Z time="2024-10-08T15:28:21Z" level=debug msg="Creating load-balancer" entryPointName=web routerName=my-site-prod-db@docker serviceName=my-site-prod-db
2024-10-08T15:28:21.701226024Z time="2024-10-08T15:28:21Z" level=debug msg="Creating server 0 http://172.17.0.3:5432" routerName=my-site-prod-db@docker serviceName=my-site-prod-db entryPointName=web serverName=0
2024-10-08T15:28:21.701237038Z time="2024-10-08T15:28:21Z" level=debug msg="child http://172.17.0.3:5432 now UP"
2024-10-08T15:28:21.701243420Z time="2024-10-08T15:28:21Z" level=debug msg="Propagating new UP status"
2024-10-08T15:28:21.701249624Z time="2024-10-08T15:28:21Z" level=debug msg="Added outgoing tracing middleware my-site-prod-db" middlewareName=tracing middlewareType=TracingForwarder entryPointName=web routerName=my-site-prod-db@docker
2024-10-08T15:28:21.701358593Z time="2024-10-08T15:28:21Z" level=debug msg="Creating middleware" routerName=my-site-prod-job-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e@docker serviceName=my-site-prod-job-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e middlewareName=pipelining middlewareType=Pipelining entryPointName=web
2024-10-08T15:28:21.701370938Z time="2024-10-08T15:28:21Z" level=debug msg="Creating load-balancer" routerName=my-site-prod-job-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e@docker serviceName=my-site-prod-job-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e entryPointName=web
2024-10-08T15:28:21.701377992Z time="2024-10-08T15:28:21Z" level=debug msg="Creating server 0 http://172.17.0.6:3000" routerName=my-site-prod-job-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e@docker serverName=0 serviceName=my-site-prod-job-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e entryPointName=web
2024-10-08T15:28:21.701404916Z time="2024-10-08T15:28:21Z" level=debug msg="child http://172.17.0.6:3000 now UP"
2024-10-08T15:28:21.701500647Z time="2024-10-08T15:28:21Z" level=debug msg="Propagating new UP status"
2024-10-08T15:28:21.701511629Z time="2024-10-08T15:28:21Z" level=debug msg="Added outgoing tracing middleware my-site-prod-job-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e" routerName=my-site-prod-job-40f5d19e16d42c4ae0091e8c4a2b4fb69aa2ab8e@docker middlewareName=tracing middlewareType=TracingForwarder entryPointName=web
2024-10-08T15:28:21.701518737Z time="2024-10-08T15:28:21Z" level=debug msg="Creating middleware" middlewareType=Pipelining entryPointName=web routerName=my-site-prod-web-prod@docker serviceName=my-site-prod-web-prod middlewareName=pipelining
2024-10-08T15:28:21.701525400Z time="2024-10-08T15:28:21Z" level=debug msg="Creating load-balancer" entryPointName=web routerName=my-site-prod-web-prod@docker serviceName=my-site-prod-web-prod
2024-10-08T15:28:21.701542484Z time="2024-10-08T15:28:21Z" level=debug msg="Creating server 0 http://172.17.0.5:3000" entryPointName=web routerName=my-site-prod-web-prod@docker serviceName=my-site-prod-web-prod serverName=0
2024-10-08T15:28:21.701550106Z time="2024-10-08T15:28:21Z" level=debug msg="child http://172.17.0.5:3000 now UP"
2024-10-08T15:28:21.701633903Z time="2024-10-08T15:28:21Z" level=debug msg="Propagating new UP status"
2024-10-08T15:28:21.701644585Z time="2024-10-08T15:28:21Z" level=debug msg="Added outgoing tracing middleware my-site-prod-web-prod" entryPointName=web routerName=my-site-prod-web-prod@docker middlewareName=tracing middlewareType=TracingForwarder
2024-10-08T15:28:21.701651443Z time="2024-10-08T15:28:21Z" level=debug msg="Creating middleware" middlewareType=Retry entryPointName=web routerName=my-site-prod-web-prod@docker middlewareName=my-site-prod-web-prod-retry@docker
2024-10-08T15:28:21.701658123Z time="2024-10-08T15:28:21Z" level=debug msg="Adding tracing to middleware" entryPointName=web routerName=my-site-prod-web-prod@docker middlewareName=my-site-prod-web-prod-retry@docker
2024-10-08T15:28:21.701664603Z time="2024-10-08T15:28:21Z" level=debug msg="Creating middleware" entryPointName=web middlewareName=traefik-internal-recovery middlewareType=Recovery
2024-10-08T15:28:21.701839357Z time="2024-10-08T15:28:21Z" level=debug msg="Creating middleware" middlewareName=traefik-internal-recovery middlewareType=Recovery entryPointName=websecure
2024-10-08T15:43:34.514842808Z time="2024-10-08T15:43:34Z" level=debug msg="Serving default certificate for request: \"\""
2024-10-08T15:43:36.849506489Z time="2024-10-08T15:43:36Z" level=debug msg="Serving default certificate for request: \"\""
2024-10-08T15:44:37.772320446Z time="2024-10-08T15:44:37Z" level=debug msg="http: TLS handshake error from 8.216.87.246:36822: EOF"
2024-10-08T16:18:22.817252645Z time="2024-10-08T16:18:22Z" level=debug msg="Serving default certificate for request: \"\""
2024-10-08T16:18:22.988133674Z time="2024-10-08T16:18:22Z" level=debug msg="Serving default certificate for request: \"\""
2024-10-08T16:19:25.680680545Z time="2024-10-08T16:19:25Z" level=debug msg="Serving default certificate for request: \"\""
2024-10-08T16:28:13.441247573Z time="2024-10-08T16:28:13Z" level=debug msg="Serving default certificate for request: \"\""
2024-10-08T16:42:20.601492235Z time="2024-10-08T16:42:20Z" level=debug msg="Serving default certificate for request: \"\""
2024-10-08T16:47:20.243334256Z time="2024-10-08T16:47:20Z" level=debug msg="Serving default certificate for request: \"\""
2024-10-08T16:58:49.075352882Z time="2024-10-08T16:58:49Z" level=debug msg="Serving default certificate for request: \"\""
2024-10-08T17:17:47.920625089Z time="2024-10-08T17:17:47Z" level=debug msg="Serving default certificate for request: \"\""
2024-10-08T17:17:48.294989855Z time="2024-10-08T17:17:48Z" level=debug msg="Serving default certificate for request: \"\""
2024-10-08T17:17:50.223648892Z time="2024-10-08T17:17:50Z" level=debug msg="Serving default certificate for request: \"\""
2024-10-08T17:17:51.698125901Z time="2024-10-08T17:17:51Z" level=debug msg="http: TLS handshake error from 167.94.146.58:36322: tls: client offered only unsupported versions: [302 301]"
2024-10-08T17:17:53.275567182Z time="2024-10-08T17:17:53Z" level=debug msg="http: TLS handshake error from 167.94.146.58:36374: tls: client offered only unsupported versions: [301]"
2024-10-08T17:17:54.842137769Z time="2024-10-08T17:17:54Z" level=debug msg="http: TLS handshake error from 167.94.146.58:36404: tls: client offered only unsupported versions: []"
2024-10-08T17:18:00.696571911Z time="2024-10-08T17:18:00Z" level=debug msg="Serving default certificate for request: \"\""
2024-10-08T17:32:44.008002451Z time="2024-10-08T17:32:44Z" level=debug msg="Serving default certificate for request: \"\""
2024-10-08T17:37:21.979375605Z time="2024-10-08T17:37:21Z" level=debug msg="Serving default certificate for request: \"\""
2024-10-08T17:41:02.435409041Z time="2024-10-08T17:41:02Z" level=debug msg="Serving default certificate for request: \"\""

I've fixed it by restarting the app:

kamal app stop -d prod
kamal app start -d prod

But in 6 hours it failed again, only restarting fixes this issue. Is there any correct way to fix this issue?

Here my deploy.yml:

service: <%= ENV['DOCKER_CONTAINER'] %>

image: <%= ENV['DOCKER_IMAGE'] %>

registry:
    #...

ssh:
  user: <%= ENV['SERVER_USER'] %>

builder:
    #...

servers:
  web:
    hosts:
      - <%= ENV['SERVER_IP'] %>
    labels:
      traefik.http.routers.my_site.rule: Host(`<%= ENV['SERVER_HOSTNAME'] %>`)
      traefik.http.routers.my_site_secure.entrypoints: websecure
      traefik.http.routers.my_site_secure.rule: Host(`<%= ENV['SERVER_HOSTNAME'] %>`)
      traefik.http.routers.my_site_secure.tls: true
      traefik.http.routers.my_site_secure.tls.certresolver: letsencrypt
      traefik.http.routers.my_site_secure.middlewares: "traefik-compress"
      traefik.http.middlewares.traefik-compress.compress: true
  job:
    hosts:
      - <%= ENV['SERVER_IP'] %>
    cmd: bundle exec rails resque:work QUEUE=*

volumes:
    #...

env:
    #...

asset_path: /rails/public/assets

accessories:
  db:
    #...

  redis:
    #...

traefik:
  options:
    publish:
      - "443:443"
    volume:
      - "/letsencrypt/acme.json:/letsencrypt/acme.json"
  args:
    entryPoints.web.address: ":80"
    entryPoints.websecure.address: ":443"
    certificatesResolvers.letsencrypt.acme.email: <%= ENV['SERVER_SSL_EMAIL'] %>
    certificatesResolvers.letsencrypt.acme.storage: "/letsencrypt/acme.json"
    certificatesResolvers.letsencrypt.acme.httpchallenge: true
    certificatesResolvers.letsencrypt.acme.httpchallenge.entrypoint: web

healthcheck:
  interval: "30s"
  max_attempts: 30
  log_lines: 100

---

Kamal v1.8.3